From b7af4017d2a5417752872d59e0c495904deb1ec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Pobo=C5=99il?= Date: Thu, 1 Oct 2020 16:45:13 +0200 Subject: [PATCH 001/289] Replaced reccomanded Docker image Previous one is 2 years old archived project, new one is automatically updated. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a63e6e42..33be1b94 100644 --- a/README.md +++ b/README.md @@ -222,7 +222,7 @@ htmlproofer --assume-extension ./_site ### Using through Docker -If you have trouble with (or don't want to) install Ruby/Nokogumbo, the command-line tool can be run through Docker. See [html-proofer-docker](https://github.com/18F/html-proofer-docker) for more information. +If you have trouble with (or don't want to) install Ruby/Nokogumbo, the command-line tool can be run through Docker. See [klakegg/html-proofer](https://hub.docker.com/r/klakegg/html-proofer) for more information. ### Using as Rack middleware From c9fb97101ab3fa56853aa44de4e3d175f85b980f Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 14 Nov 2020 10:25:37 +0000 Subject: [PATCH 002/289] it's ok --- CODEOWNERS | 1 - 1 file changed, 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 35056083..53a4a89d 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -3,4 +3,3 @@ # These owners will be the default owners for everything in the repo. * @gjtorikian -* @fulldecent From 094d2173009496d63f5598bd730692067326f66f Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 14 Nov 2020 13:10:19 +0000 Subject: [PATCH 003/289] support cache for internal links --- lib/html-proofer/cache.rb | 6 +- lib/html-proofer/check.rb | 29 +++- lib/html-proofer/check/links.rb | 51 ++++--- lib/html-proofer/runner.rb | 77 +++++++++-- lib/html-proofer/url_validator.rb | 15 +-- spec/html-proofer/cache_spec.rb | 54 +++++++- .../{.new_url.log => .new_external_url.log} | 0 .../links/broken_internal_hashes/file.html | 9 ++ .../links/broken_internal_hashes/file1.html | 9 ++ .../links/broken_internal_hashes/file2.html | 9 ++ .../links/broken_internal_hashes/file3.html | 10 ++ .../fixtures/links/broken_internal_link.html | 9 ++ .../cache_log_level_error_type_directory_.yml | 28 ++++ .../cache_log_level_error_type_links_.yml | 28 ++++ .../cache_log_level_error_type_links_.yml | 124 ++++++++++++++++++ .../cache_log_level_error_type_links_.yml | 75 +++++++++++ 16 files changed, 478 insertions(+), 55 deletions(-) rename spec/html-proofer/fixtures/cache/{.new_url.log => .new_external_url.log} (100%) create mode 100644 spec/html-proofer/fixtures/links/broken_internal_hashes/file.html create mode 100644 spec/html-proofer/fixtures/links/broken_internal_hashes/file1.html create mode 100644 spec/html-proofer/fixtures/links/broken_internal_hashes/file2.html create mode 100644 spec/html-proofer/fixtures/links/broken_internal_hashes/file3.html create mode 100644 spec/html-proofer/fixtures/vcr_cassettes/links/broken_internal_hashes_cache_timeframe_30d_cache_file_new_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_directory_.yml create mode 100644 spec/html-proofer/fixtures/vcr_cassettes/test_cache_timeframe_30d_cache_file_new_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml create mode 100644 spec/html-proofer/fixtures/vcr_cassettes/www_github_com_www_google_com_cache_timeframe_30d_cache_file_new_external_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml diff --git a/lib/html-proofer/cache.rb b/lib/html-proofer/cache.rb index 6351dd23..406fb46b 100644 --- a/lib/html-proofer/cache.rb +++ b/lib/html-proofer/cache.rb @@ -59,6 +59,8 @@ def parsed_timeframe(timeframe) end def add(url, filenames, status, msg = '') + return unless use_cache? + data = { time: @cache_time, filenames: filenames, @@ -115,8 +117,8 @@ def load? @load.nil? end - def retrieve_urls(external_urls) - urls_to_check = detect_url_changes(external_urls) + def retrieve_urls(urls) + urls_to_check = detect_url_changes(urls) @cache_log.each_pair do |url, cache| if within_timeframe?(cache['time']) next if cache['message'].empty? # these were successes to skip diff --git a/lib/html-proofer/check.rb b/lib/html-proofer/check.rb index 60e9a749..c69b9774 100644 --- a/lib/html-proofer/check.rb +++ b/lib/html-proofer/check.rb @@ -3,15 +3,17 @@ module HTMLProofer # Mostly handles issue management and collecting of external URLs. class Check - attr_reader :node, :html, :element, :src, :path, :options, :issues, :external_urls + attr_reader :node, :html, :element, :src, :path, :options, :issues, :internal_urls, :external_urls - def initialize(src, path, html, logger, options) + def initialize(src, path, html, logger, cache, options) @src = src @path = path @html = remove_ignored(html) @logger = logger + @cache = cache @options = options @issues = [] + @internal_urls = {} @external_urls = {} end @@ -26,19 +28,32 @@ def run def add_issue(desc, line: nil, status: -1, content: nil) @issues << Issue.new(@path, desc, line: line, status: status, content: content) + false + end + + def add_to_internal_urls(url) + add_path_for_url(url, type: :internal) end def add_to_external_urls(url) return if @external_urls[url] - add_path_for_url(url) + add_path_for_url(url, type: :external) end - def add_path_for_url(url) - if @external_urls[url] - @external_urls[url] << @path + def add_path_for_url(url, type: :external) + if type == :external + if @external_urls[url] + @external_urls[url] << @path + else + @external_urls[url] = [@path] + end else - @external_urls[url] = [@path] + if @internal_urls[url] + @internal_urls[url] << @path + else + @internal_urls[url] = [@path] + end end end diff --git a/lib/html-proofer/check/links.rb b/lib/html-proofer/check/links.rb index 02535a29..2bf2f994 100644 --- a/lib/html-proofer/check/links.rb +++ b/lib/html-proofer/check/links.rb @@ -3,6 +3,8 @@ class LinkCheck < ::HTMLProofer::Check include HTMLProofer::Utils + attr_reader :link, :line, :content + def missing_href? blank?(@link.href) && blank?(@link.name) && blank?(@link.id) end @@ -14,8 +16,8 @@ def placeholder? def run @html.css('a, link').each do |node| @link = create_element(node) - line = node.line - content = node.to_s + @line = node.line + @content = node.to_s next if @link.ignore? @@ -48,24 +50,31 @@ def run # we need to skip these for now; although the domain main be valid, # curl/Typheous inaccurately return 404s for some links. cc https://git.io/vyCFx next if @link.respond_to?(:rel) && @link.rel == 'dns-prefetch' - add_to_external_urls(@link.href) next - elsif @link.internal? && !@link.exists? - add_issue("internally linking to #{@link.href}, which does not exist", line: line, content: content) + elsif @link.internal? + if !@link.exists? + add_issue("internally linking to #{@link.href}, which does not exist", line: line, content: content) + else + add_to_internal_urls(@link.href) + end end + end - # does the local directory have a trailing slash? - if @link.unslashed_directory? @link.absolute_path - add_issue("internally linking to a directory #{@link.absolute_path} without trailing slash", line: line, content: content) - next - end + external_urls + end - # verify the target hash - handle_hash(@link, line, content) if @link.hash + def check_internal_link + # does the local directory have a trailing slash? + if @link.unslashed_directory? @link.absolute_path + add_issue("internally linking to a directory #{@link.absolute_path} without trailing slash", line: line, content: content) + return false end - external_urls + # verify the target hash + return handle_hash(@link, line, content) if @link.hash + + true end def check_schemes(link, line, content) @@ -95,19 +104,23 @@ def handle_tel(link, line, content) def handle_hash(link, line, content) if link.internal? && !hash_check(link.html, link.hash) - add_issue("linking to internal hash ##{link.hash} that does not exist", line: line, content: content) + return add_issue("linking to internal hash ##{link.hash} that does not exist", line: line, content: content) elsif link.external? - external_link_check(link, line, content) + return external_link_check(link, line, content) end + + true end def external_link_check(link, line, content) if !link.exists? - add_issue("trying to find hash of #{link.href}, but #{link.absolute_path} does not exist", line: line, content: content) + return add_issue("trying to find hash of #{link.href}, but #{link.absolute_path} does not exist", line: line, content: content) else - target_html = create_nokogiri link.absolute_path - add_issue("linking to #{link.href}, but #{link.hash} does not exist", line: line, content: content) unless hash_check target_html, link.hash + target_html = create_nokogiri(link.absolute_path) + return add_issue("linking to #{link.href}, but #{link.hash} does not exist", line: line, content: content) unless hash_check(target_html, link.hash) end + + true end def hash_check(html, href_hash) @@ -148,7 +161,7 @@ def check_sri(line, content) class XpathFunctions def case_sensitive_equals(node_set, str_to_match) - node_set.find_all { |node| node.to_s. == str_to_match.to_s } + node_set.find_all { |node| node.to_s.== str_to_match.to_s } end end end diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index 7a59737f..419db073 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -4,7 +4,7 @@ module HTMLProofer class Runner include HTMLProofer::Utils - attr_reader :options, :external_urls, :failures + attr_reader :options, :internal_urls, :external_urls, :failures def initialize(src, opts = {}) @src = src @@ -20,6 +20,8 @@ def initialize(src, opts = {}) @type = @options.delete(:type) @logger = HTMLProofer::Log.new(@options[:log_level]) + @cache = Cache.new(@logger, @options[:cache]) + @internal_link_checks = [] # Add swap patterns for internal domains unless @options[:internal_domains].empty? @@ -30,6 +32,8 @@ def initialize(src, opts = {}) end end + @internal_urls = {} + @external_urls = {} @failures = [] @before_request = [] end @@ -59,17 +63,16 @@ def check_list_of_links end end @external_urls = Hash[*@src.map { |s| [s, nil] }.flatten] - validate_urls + validate_external_urls end # Collects any external URLs found in a directory of files. Also collectes # every failed test from process_files. # Sends the external URLs to Typhoeus for batch processing. def check_files - @external_urls = {} - process_files.each do |item| - @external_urls.merge!(item[:external_urls]) + @external_urls.merge!(item[:external_urls]) #{ |k, o, (n, _)| o << n } + @internal_urls.merge!(item[:internal_urls]) { |k, o, (n, _)| o << n } @failures.concat(item[:failures]) end @@ -78,9 +81,12 @@ def check_files # just not run those other checks at all. if @options[:external_only] @failures = [] - validate_urls + validate_external_urls elsif !@options[:disable_external] - validate_urls + validate_external_urls + validate_internal_urls + else + validate_internal_urls end end @@ -94,15 +100,19 @@ def process_files end def check_parsed(html, path) - result = { external_urls: {}, failures: [] } + result = { internal_urls: {}, external_urls: {}, failures: [] } @src = [@src] if @type == :file @src.each do |src| checks.each do |klass| @logger.log :debug, "Checking #{klass.to_s.downcase} on #{path} ..." - check = Object.const_get(klass).new(src, path, html, @logger, @options) + check = Object.const_get(klass).new(src, path, html, @logger, @cache, @options) check.run + + @internal_link_checks << check if klass == 'LinkCheck' && !check.link.nil? + + result[:internal_urls].merge!(check.internal_urls) external_urls = check.external_urls external_urls = Hash[check.external_urls.map { |url, file| [swap(url, @options[:url_swap]), file] }] if @options[:url_swap] result[:external_urls].merge!(external_urls) @@ -112,17 +122,52 @@ def check_parsed(html, path) result end + def add_path_for_url(url, type: :external) + if type == :external + if @external_urls[url] + @external_urls[url] << @path + else + @external_urls[url] = [@path] + end + else + if @internal_urls[url] + @internal_urls[url] << @path + else + @internal_urls[url] = [@path] + end + end + end + def check_path(path) - check_parsed create_nokogiri(path), path + check_parsed(create_nokogiri(path), path) end - def validate_urls - url_validator = HTMLProofer::UrlValidator.new(@logger, @external_urls, @options) + def validate_external_urls + url_validator = HTMLProofer::UrlValidator.new(@logger, @cache, @external_urls, @options) url_validator.before_request = @before_request @failures.concat(url_validator.run) @external_urls = url_validator.external_urls end + def validate_internal_urls + if @cache.use_cache? + urls_to_check = load_internal_cache + urls_to_check.each_pair do |url, filenames| + @internal_link_checks.each do |check| + if url == check.link.href + (result = check.check_internal_link) + code = result ? 200 : 404 + @cache.add(url, filenames, code, '') # TODO: blank msg for now + break + end + end + end + @cache.write + else + @internal_link_checks.each(&:check_internal_link) + end + end + def files @files ||= if @type == :directory @src.map do |src| @@ -191,5 +236,13 @@ def before_request(&block) @before_request << block if block_given? @before_request end + + def load_internal_cache + urls_to_check = @cache.retrieve_urls(@internal_urls) + cache_text = pluralize(urls_to_check.count, 'internal link', 'internal links') + @logger.log :info, "Found #{cache_text} in the cache..." + + urls_to_check + end end end diff --git a/lib/html-proofer/url_validator.rb b/lib/html-proofer/url_validator.rb index b4f0a0d2..59e8bf5f 100644 --- a/lib/html-proofer/url_validator.rb +++ b/lib/html-proofer/url_validator.rb @@ -12,13 +12,13 @@ class UrlValidator attr_reader :external_urls attr_writer :before_request - def initialize(logger, external_urls, options) + def initialize(logger, cache, external_urls, options) @logger = logger @external_urls = external_urls @failed_tests = [] @options = options @hydra = Typhoeus::Hydra.new(@options[:hydra]) - @cache = Cache.new(@logger, @options[:cache]) + @cache = cache @before_request = [] end @@ -26,7 +26,7 @@ def run @external_urls = remove_query_values if @cache.use_cache? - urls_to_check = load_cache + urls_to_check = @cache.retrieve_urls(@external_urls) external_link_checker(urls_to_check) @cache.write else @@ -74,15 +74,6 @@ def extract_domain_path(uri) uri.host + uri.path end - def load_cache - cache_count = @cache.size - cache_text = pluralize(cache_count, 'link', 'links') - - @logger.log :info, "Found #{cache_text} in the cache..." - - @cache.retrieve_urls(@external_urls) - end - # Proofer runs faster if we pull out all the external URLs and run the checks # at the end. Otherwise, we're halting the consuming process for every file during # `process_files`. diff --git a/spec/html-proofer/cache_spec.rb b/spec/html-proofer/cache_spec.rb index 54cb54c7..9fed2ba0 100644 --- a/spec/html-proofer/cache_spec.rb +++ b/spec/html-proofer/cache_spec.rb @@ -90,7 +90,7 @@ def read_cache(cache_file) it 'knows how to write to cache' do broken_link_external_filepath = "#{FIXTURES_DIR}/links/broken_link_external.html" - expect_any_instance_of(HTMLProofer::Cache).to receive(:write) + expect_any_instance_of(HTMLProofer::Cache).to receive(:write).twice # once for internal, once for external run_proofer(broken_link_external_filepath, :file, cache: { timeframe: '30d', cache_file: cache_file_name }.merge(default_cache_options)) log = read_cache(cache_file) @@ -144,8 +144,8 @@ def read_cache(cache_file) end end - context 'new url added' do - let(:cache_file_name) { '.new_url.log' } + context 'new external url added' do + let(:cache_file_name) { '.new_external_url.log' } it 'does write file if a new URL is added' do # this is frozen to within 7 days of the log new_time = Time.local(2015, 10, 20, 12, 0, 0) @@ -162,6 +162,54 @@ def read_cache(cache_file) end end + context 'new internal url added' do + let(:cache_file_name) { '.new_internal_url.log' } + it 'does write file if a new relative URL 200 is added' do + # this is frozen to within 7 days of the log + new_time = Time.local(2015, 10, 20, 12, 0, 0) + Timecop.freeze(new_time) + + expect_any_instance_of(HTMLProofer::Cache).to receive(:write) + root_link = "#{FIXTURES_DIR}/links/root_link/root_link.html" + expect_any_instance_of(HTMLProofer::Cache).to receive(:add).with("/", ["spec/html-proofer/fixtures/links/root_link/root_link.html"], 200, "") + + # we expect one new link to be added because it's within the 30d time frame + run_proofer(root_link, :file, disable_external: true, cache: { timeframe: '30d', cache_file: cache_file_name }.merge(default_cache_options)) + + Timecop.return + end + + it 'does write file if a new relative URL 404 is added' do + # this is frozen to within 7 days of the log + new_time = Time.local(2015, 10, 20, 12, 0, 0) + Timecop.freeze(new_time) + + expect_any_instance_of(HTMLProofer::Cache).to receive(:write) + root_link = "#{FIXTURES_DIR}/links/broken_internal_link.html" + expect_any_instance_of(HTMLProofer::Cache).to receive(:add).with("#noHash", ["spec/html-proofer/fixtures/links/broken_internal_link.html"], 404, "") + + # we expect one new link to be added because it's within the 30d time frame + run_proofer(root_link, :file, disable_external: true, cache: { timeframe: '30d', cache_file: cache_file_name }.merge(default_cache_options)) + + Timecop.return + end + + it 'does writes file once if a new relative URL 404 hash is detected multiple times' do + # this is frozen to within 7 days of the log + new_time = Time.local(2015, 10, 20, 12, 0, 0) + Timecop.freeze(new_time) + + expect_any_instance_of(HTMLProofer::Cache).to receive(:write) + root_link = "#{FIXTURES_DIR}/links/broken_internal_hashes" + expect_any_instance_of(HTMLProofer::Cache).to receive(:add).with("file.html#noHash", ["spec/html-proofer/fixtures/links/broken_internal_hashes/file1.html", "spec/html-proofer/fixtures/links/broken_internal_hashes/file2.html", "spec/html-proofer/fixtures/links/broken_internal_hashes/file3.html"], 404, "").once + + # we expect one new link to be added because it's within the 30d time frame + run_proofer(root_link, :directory, disable_external: true, cache: { timeframe: '30d', cache_file: cache_file_name }.merge(default_cache_options)) + + Timecop.return + end + end + context 'recheck failure' do let(:cache_file_name) { '.recheck_failure.log' } it 'does recheck failures, regardless of cache' do diff --git a/spec/html-proofer/fixtures/cache/.new_url.log b/spec/html-proofer/fixtures/cache/.new_external_url.log similarity index 100% rename from spec/html-proofer/fixtures/cache/.new_url.log rename to spec/html-proofer/fixtures/cache/.new_external_url.log diff --git a/spec/html-proofer/fixtures/links/broken_internal_hashes/file.html b/spec/html-proofer/fixtures/links/broken_internal_hashes/file.html new file mode 100644 index 00000000..abb490d6 --- /dev/null +++ b/spec/html-proofer/fixtures/links/broken_internal_hashes/file.html @@ -0,0 +1,9 @@ + + + + +

Blah blah blah.

+ + + + diff --git a/spec/html-proofer/fixtures/links/broken_internal_hashes/file1.html b/spec/html-proofer/fixtures/links/broken_internal_hashes/file1.html new file mode 100644 index 00000000..e41f9eab --- /dev/null +++ b/spec/html-proofer/fixtures/links/broken_internal_hashes/file1.html @@ -0,0 +1,9 @@ + + + + +

Blah blah blah. Not a real hash!

+ + + + diff --git a/spec/html-proofer/fixtures/links/broken_internal_hashes/file2.html b/spec/html-proofer/fixtures/links/broken_internal_hashes/file2.html new file mode 100644 index 00000000..e41f9eab --- /dev/null +++ b/spec/html-proofer/fixtures/links/broken_internal_hashes/file2.html @@ -0,0 +1,9 @@ + + + + +

Blah blah blah. Not a real hash!

+ + + + diff --git a/spec/html-proofer/fixtures/links/broken_internal_hashes/file3.html b/spec/html-proofer/fixtures/links/broken_internal_hashes/file3.html new file mode 100644 index 00000000..59de143b --- /dev/null +++ b/spec/html-proofer/fixtures/links/broken_internal_hashes/file3.html @@ -0,0 +1,10 @@ + + + + +

Blah blah blah. Not a real hash!

+ + + + + diff --git a/spec/html-proofer/fixtures/links/broken_internal_link.html b/spec/html-proofer/fixtures/links/broken_internal_link.html index e69de29b..e9ad41b0 100644 --- a/spec/html-proofer/fixtures/links/broken_internal_link.html +++ b/spec/html-proofer/fixtures/links/broken_internal_link.html @@ -0,0 +1,9 @@ + + + + +

Blah blah blah. Not a real hash!

+ + + + diff --git a/spec/html-proofer/fixtures/vcr_cassettes/links/broken_internal_hashes_cache_timeframe_30d_cache_file_new_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_directory_.yml b/spec/html-proofer/fixtures/vcr_cassettes/links/broken_internal_hashes_cache_timeframe_30d_cache_file_new_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_directory_.yml new file mode 100644 index 00000000..e20f7909 --- /dev/null +++ b/spec/html-proofer/fixtures/vcr_cassettes/links/broken_internal_hashes_cache_timeframe_30d_cache_file_new_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_directory_.yml @@ -0,0 +1,28 @@ +--- +http_interactions: +- request: + method: head + uri: https://fofoadsad.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.16.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: https://fofoadsad.com + recorded_at: Sat, 14 Nov 2020 12:38:05 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html-proofer/fixtures/vcr_cassettes/test_cache_timeframe_30d_cache_file_new_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml b/spec/html-proofer/fixtures/vcr_cassettes/test_cache_timeframe_30d_cache_file_new_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml new file mode 100644 index 00000000..5a209743 --- /dev/null +++ b/spec/html-proofer/fixtures/vcr_cassettes/test_cache_timeframe_30d_cache_file_new_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml @@ -0,0 +1,28 @@ +--- +http_interactions: +- request: + method: head + uri: "/test" + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.16.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: "/test" + recorded_at: Tue, 20 Oct 2015 11:00:00 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html-proofer/fixtures/vcr_cassettes/www_github_com_www_google_com_cache_timeframe_30d_cache_file_new_external_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml b/spec/html-proofer/fixtures/vcr_cassettes/www_github_com_www_google_com_cache_timeframe_30d_cache_file_new_external_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml new file mode 100644 index 00000000..b2d879c9 --- /dev/null +++ b/spec/html-proofer/fixtures/vcr_cassettes/www_github_com_www_google_com_cache_timeframe_30d_cache_file_new_external_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml @@ -0,0 +1,124 @@ +--- +http_interactions: +- request: + method: head + uri: www.google.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.16.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - text/html; charset=UTF-8 + P3P: + - CP="This is not a P3P policy! See g.co/p3phelp for more info." + Date: + - Sat, 14 Nov 2020 13:08:53 GMT + Server: + - gws + X-XSS-Protection: + - '0' + X-Frame-Options: + - SAMEORIGIN + Transfer-Encoding: + - chunked + Expires: + - Sat, 14 Nov 2020 13:08:53 GMT + Cache-Control: + - private + Set-Cookie: + - NID=204=n2o4nW5Slr5jAF7aFTqm75c53g5olObdl477Av3bo7II7EcdcOavGEN0xfoqIE9SmS28SXX1mS4U1mtAPDHivjR_nqm3iOo3FohiXQPqvgbhYt77OwDijagR15oA9LGvrCL2AKcg7Z4vJQHGuWnFAjKNcgIydJXnWVsw3cDpheQ; + expires=Sun, 16-May-2021 13:08:53 GMT; path=/; domain=.google.com; HttpOnly + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: www.google.com + recorded_at: Tue, 20 Oct 2015 11:00:00 GMT +- request: + method: head + uri: www.github.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.16.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: OK + headers: + date: + - Sat, 14 Nov 2020 13:08:53 GMT + content-type: + - text/html; charset=utf-8 + server: + - GitHub.com + status: + - 200 OK + vary: + - X-PJAX, Accept-Encoding, Accept, X-Requested-With, Accept-Encoding + etag: + - W/"1b648d5d0adcbc2b2a614ac2299ecb44" + cache-control: + - max-age=0, private, must-revalidate + strict-transport-security: + - max-age=31536000; includeSubdomains; preload + x-frame-options: + - deny + x-content-type-options: + - nosniff + x-xss-protection: + - 1; mode=block + referrer-policy: + - origin-when-cross-origin, strict-origin-when-cross-origin + expect-ct: + - max-age=2592000, report-uri="https://api.github.com/_private/browser/errors" + content-security-policy: + - 'default-src ''none''; base-uri ''self''; block-all-mixed-content; connect-src + ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com + github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com + github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com + cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com; font-src + github.githubassets.com; form-action ''self'' github.com gist.github.com; + frame-ancestors ''none''; frame-src render.githubusercontent.com; img-src + ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com + github-cloud.s3.amazonaws.com *.githubusercontent.com customer-stories-feed.github.com + spotlights-feed.github.com; manifest-src ''self''; media-src github.githubassets.com; + script-src github.githubassets.com; style-src ''unsafe-inline'' github.githubassets.com; + worker-src github.com/socket-worker.js gist.github.com/socket-worker.js' + Set-Cookie: + - _gh_sess=Wt7bflZolRuy2%2B7MF18AiHnIsTO6zjRg0DdAYel%2FBa2Oz0uL8FWjbuiuj2BDWZbtnsULUyHpfV2sl8Bbb1Ma7qcKL6nxeauwPv2a7UJPGb83GOryTsk7TRwCvDsksxgVz%2FzgydxpUfJTCN%2Bmki47tCPIVoYgykK9%2Bx5GeFd%2B6ill0MYtz4eyHr4F%2BDAheiD%2F43ZGl6y5C7MaksIkSp%2BxGUCTlWvf6GYBVvx4Gr0jWnQDf%2BqKC4k%2B%2FqG3LUOUJQUAMCY8PUwobRT7Tg3MPhztyw%3D%3D--kU5Qj36%2FErMcNmZt--Ls0XYiR2muSIOo22V%2BjQcQ%3D%3D; + Path=/; HttpOnly; Secure; SameSite=Lax + - _octo=GH1.1.1685091136.1605359333; Path=/; Domain=github.com; Expires=Sun, + 14 Nov 2021 13:08:53 GMT; Secure; SameSite=Lax + - logged_in=no; Path=/; Domain=github.com; Expires=Sun, 14 Nov 2021 13:08:53 + GMT; HttpOnly; Secure; SameSite=Lax + Accept-Ranges: + - bytes + X-GitHub-Request-Id: + - E6D3:F0F7:18D4C75:233FF45:5FAFD6E5 + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: https://github.com/ + recorded_at: Tue, 20 Oct 2015 11:00:00 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html-proofer/fixtures/vcr_cassettes/www_github_com_www_google_com_cache_timeframe_30d_cache_file_new_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml b/spec/html-proofer/fixtures/vcr_cassettes/www_github_com_www_google_com_cache_timeframe_30d_cache_file_new_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml index 03fd8759..edc3ab4e 100644 --- a/spec/html-proofer/fixtures/vcr_cassettes/www_github_com_www_google_com_cache_timeframe_30d_cache_file_new_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml +++ b/spec/html-proofer/fixtures/vcr_cassettes/www_github_com_www_google_com_cache_timeframe_30d_cache_file_new_url_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml @@ -47,4 +47,79 @@ http_interactions: adapter_metadata: effective_url: http://www.google.com/ recorded_at: Tue, 20 Oct 2015 16:00:00 GMT +- request: + method: head + uri: www.github.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.16.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: OK + headers: + date: + - Sat, 14 Nov 2020 13:03:02 GMT + content-type: + - text/html; charset=utf-8 + server: + - GitHub.com + status: + - 200 OK + vary: + - X-PJAX, Accept-Encoding, Accept, X-Requested-With, Accept-Encoding + etag: + - W/"d9e1b416b6d49de7660d8ce7ef2f12c6" + cache-control: + - max-age=0, private, must-revalidate + strict-transport-security: + - max-age=31536000; includeSubdomains; preload + x-frame-options: + - deny + x-content-type-options: + - nosniff + x-xss-protection: + - 1; mode=block + referrer-policy: + - origin-when-cross-origin, strict-origin-when-cross-origin + expect-ct: + - max-age=2592000, report-uri="https://api.github.com/_private/browser/errors" + content-security-policy: + - 'default-src ''none''; base-uri ''self''; block-all-mixed-content; connect-src + ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com + github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com + github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com + cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com; font-src + github.githubassets.com; form-action ''self'' github.com gist.github.com; + frame-ancestors ''none''; frame-src render.githubusercontent.com; img-src + ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com + github-cloud.s3.amazonaws.com *.githubusercontent.com customer-stories-feed.github.com + spotlights-feed.github.com; manifest-src ''self''; media-src github.githubassets.com; + script-src github.githubassets.com; style-src ''unsafe-inline'' github.githubassets.com; + worker-src github.com/socket-worker.js gist.github.com/socket-worker.js' + Set-Cookie: + - _gh_sess=R2fIJ1lE05su3AjGCtAp1m63uq7MnwtYhFxIgIzTeI3D4VovaOUs1RfBlDo0LxtaESj4rfu40XvQGPW7UmTOCN7lpT5Wi3lEcbcd4nbzqCsJaWBTnspuUO4HZFAL2oIgSGLeT15ORU7xM0nSuqAH%2BaCKcMjQ2lFTGD5PcOjTm%2Bg2ZhUHhe%2B88mk2Dz%2Bfy2kW%2FiwtUTTcBD3Apw2uwq%2FX90zZK3bLubXIZr4mcqokKF8yR80hySqrnGHYiN4s9mx6oESKpJfzKSKHvwRgRtmnUQ%3D%3D--Uwf7K2SM2YHvlCEI--Ic1O6W%2Ff%2F5HCwuS90fLqeQ%3D%3D; + Path=/; HttpOnly; Secure; SameSite=Lax + - _octo=GH1.1.2031775501.1605358981; Path=/; Domain=github.com; Expires=Sun, + 14 Nov 2021 13:03:01 GMT; Secure; SameSite=Lax + - logged_in=no; Path=/; Domain=github.com; Expires=Sun, 14 Nov 2021 13:03:01 + GMT; HttpOnly; Secure; SameSite=Lax + Accept-Ranges: + - bytes + X-GitHub-Request-Id: + - E10C:DE96:407E4:686DD:5FAFD585 + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: https://github.com/ + recorded_at: Tue, 20 Oct 2015 11:00:00 GMT recorded_with: VCR 2.9.3 From ac382e78b158c078d3fe7865d0384f641aa5441b Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 14 Nov 2020 15:21:58 +0000 Subject: [PATCH 004/289] fix tests --- lib/html-proofer/check.rb | 26 +++++--------- lib/html-proofer/check/links.rb | 35 ++++++++++++------- lib/html-proofer/runner.rb | 57 ++++++++++++++----------------- spec/html-proofer/element_spec.rb | 2 +- 4 files changed, 58 insertions(+), 62 deletions(-) diff --git a/lib/html-proofer/check.rb b/lib/html-proofer/check.rb index c69b9774..6ab8c17a 100644 --- a/lib/html-proofer/check.rb +++ b/lib/html-proofer/check.rb @@ -31,29 +31,21 @@ def add_issue(desc, line: nil, status: -1, content: nil) false end - def add_to_internal_urls(url) - add_path_for_url(url, type: :internal) + def add_to_internal_urls(url, internal_url) + if @internal_urls[url] + @internal_urls[url] << internal_url + else + @internal_urls[url] = [internal_url] + end end def add_to_external_urls(url) return if @external_urls[url] - add_path_for_url(url, type: :external) - end - - def add_path_for_url(url, type: :external) - if type == :external - if @external_urls[url] - @external_urls[url] << @path - else - @external_urls[url] = [@path] - end + if @external_urls[url] + @external_urls[url] << @path else - if @internal_urls[url] - @internal_urls[url] << @path - else - @internal_urls[url] = [@path] - end + @external_urls[url] = [@path] end end diff --git a/lib/html-proofer/check/links.rb b/lib/html-proofer/check/links.rb index 2bf2f994..fb616e8c 100644 --- a/lib/html-proofer/check/links.rb +++ b/lib/html-proofer/check/links.rb @@ -3,8 +3,6 @@ class LinkCheck < ::HTMLProofer::Check include HTMLProofer::Utils - attr_reader :link, :line, :content - def missing_href? blank?(@link.href) && blank?(@link.name) && blank?(@link.id) end @@ -16,8 +14,8 @@ def placeholder? def run @html.css('a, link').each do |node| @link = create_element(node) - @line = node.line - @content = node.to_s + line = node.line + content = node.to_s next if @link.ignore? @@ -50,13 +48,14 @@ def run # we need to skip these for now; although the domain main be valid, # curl/Typheous inaccurately return 404s for some links. cc https://git.io/vyCFx next if @link.respond_to?(:rel) && @link.rel == 'dns-prefetch' + add_to_external_urls(@link.href) next elsif @link.internal? if !@link.exists? add_issue("internally linking to #{@link.href}, which does not exist", line: line, content: content) else - add_to_internal_urls(@link.href) + add_to_internal_urls(@link.href, InternalLink.new(@link, @path, line, content)) end end end @@ -64,15 +63,15 @@ def run external_urls end - def check_internal_link + def check_internal_link(link, line, content) # does the local directory have a trailing slash? - if @link.unslashed_directory? @link.absolute_path - add_issue("internally linking to a directory #{@link.absolute_path} without trailing slash", line: line, content: content) + if link.unslashed_directory?(link.absolute_path) + add_issue("internally linking to a directory #{link.absolute_path} without trailing slash", line: line, content: content) return false end # verify the target hash - return handle_hash(@link, line, content) if @link.hash + return handle_hash(link, line, content) if link.hash true end @@ -103,7 +102,7 @@ def handle_tel(link, line, content) end def handle_hash(link, line, content) - if link.internal? && !hash_check(link.html, link.hash) + if link.internal? && !hash_exists?(link.html, link.hash) return add_issue("linking to internal hash ##{link.hash} that does not exist", line: line, content: content) elsif link.external? return external_link_check(link, line, content) @@ -117,13 +116,13 @@ def external_link_check(link, line, content) return add_issue("trying to find hash of #{link.href}, but #{link.absolute_path} does not exist", line: line, content: content) else target_html = create_nokogiri(link.absolute_path) - return add_issue("linking to #{link.href}, but #{link.hash} does not exist", line: line, content: content) unless hash_check(target_html, link.hash) + return add_issue("linking to #{link.href}, but #{link.hash} does not exist", line: line, content: content) unless hash_exists?(target_html, link.hash) end true end - def hash_check(html, href_hash) + def hash_exists?(html, href_hash) decoded_href_hash = Addressable::URI.unescape(href_hash) fragment_ids = [href_hash, decoded_href_hash] # https://www.w3.org/TR/html5/single-page.html#scroll-to-fragid @@ -164,4 +163,16 @@ def case_sensitive_equals(node_set, str_to_match) node_set.find_all { |node| node.to_s.== str_to_match.to_s } end end + + class InternalLink + attr_reader :link, :href, :path, :line, :content + + def initialize(link, path, line, content) + @link = link + @href = @link.href + @path = path + @line = line + @content = content + end + end end diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index 419db073..a7e9329a 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -21,7 +21,7 @@ def initialize(src, opts = {}) @type = @options.delete(:type) @logger = HTMLProofer::Log.new(@options[:log_level]) @cache = Cache.new(@logger, @options[:cache]) - @internal_link_checks = [] + @internal_link_checks = nil # Add swap patterns for internal domains unless @options[:internal_domains].empty? @@ -33,6 +33,7 @@ def initialize(src, opts = {}) end @internal_urls = {} + @internal_urls_to_paths = {} @external_urls = {} @failures = [] @before_request = [] @@ -71,8 +72,7 @@ def check_list_of_links # Sends the external URLs to Typhoeus for batch processing. def check_files process_files.each do |item| - @external_urls.merge!(item[:external_urls]) #{ |k, o, (n, _)| o << n } - @internal_urls.merge!(item[:internal_urls]) { |k, o, (n, _)| o << n } + @external_urls.merge!(item[:external_urls]) @failures.concat(item[:failures]) end @@ -100,7 +100,7 @@ def process_files end def check_parsed(html, path) - result = { internal_urls: {}, external_urls: {}, failures: [] } + result = { external_urls: {}, failures: [] } @src = [@src] if @type == :file @@ -110,9 +110,18 @@ def check_parsed(html, path) check = Object.const_get(klass).new(src, path, html, @logger, @cache, @options) check.run - @internal_link_checks << check if klass == 'LinkCheck' && !check.link.nil? + if klass == 'LinkCheck' + @internal_link_checks = check + check.internal_urls.each_pair do |url, internal_urls| + if @internal_urls_to_paths[url] + @internal_urls_to_paths[url].concat(internal_urls.map(&:path)) + else + @internal_urls_to_paths[url] = internal_urls.map(&:path) + end + end + @internal_urls.merge!(check.internal_urls) + end - result[:internal_urls].merge!(check.internal_urls) external_urls = check.external_urls external_urls = Hash[check.external_urls.map { |url, file| [swap(url, @options[:url_swap]), file] }] if @options[:url_swap] result[:external_urls].merge!(external_urls) @@ -122,22 +131,6 @@ def check_parsed(html, path) result end - def add_path_for_url(url, type: :external) - if type == :external - if @external_urls[url] - @external_urls[url] << @path - else - @external_urls[url] = [@path] - end - else - if @internal_urls[url] - @internal_urls[url] << @path - else - @internal_urls[url] = [@path] - end - end - end - def check_path(path) check_parsed(create_nokogiri(path), path) end @@ -152,20 +145,20 @@ def validate_external_urls def validate_internal_urls if @cache.use_cache? urls_to_check = load_internal_cache - urls_to_check.each_pair do |url, filenames| - @internal_link_checks.each do |check| - if url == check.link.href - (result = check.check_internal_link) - code = result ? 200 : 404 - @cache.add(url, filenames, code, '') # TODO: blank msg for now - break - end - end + + urls_to_check.each_pair do |url, internal_urls| + result = @internal_link_checks.check_internal_link(internal_urls.first.link, internal_urls.first.line, internal_urls.first.content) + code = result ? 200 : 404 + @cache.add(url, @internal_urls_to_paths[url], code, '') # TODO: blank msg for now end @cache.write else - @internal_link_checks.each(&:check_internal_link) + @internal_urls.values.flatten.each do |internal_url| + @internal_link_checks.check_internal_link(internal_url.link, internal_url.line, internal_url.content) + end end + + @failures.concat(@internal_link_checks.issues) end def files diff --git a/spec/html-proofer/element_spec.rb b/spec/html-proofer/element_spec.rb index c5ce7013..51936b13 100644 --- a/spec/html-proofer/element_spec.rb +++ b/spec/html-proofer/element_spec.rb @@ -4,7 +4,7 @@ describe HTMLProofer::Element do before(:each) do - @check = HTMLProofer::Check.new('', '', Nokogiri::HTML5(''), nil, HTMLProofer::Configuration::PROOFER_DEFAULTS) + @check = HTMLProofer::Check.new('', '', Nokogiri::HTML5(''), nil, nil, HTMLProofer::Configuration::PROOFER_DEFAULTS) end describe '#initialize' do From cbcd0aab5255ad883f1c07a4fa8bce64e993c39d Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 14 Nov 2020 15:57:53 +0000 Subject: [PATCH 005/289] Remove needless duplication --- lib/html-proofer/runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index a7e9329a..df69d59a 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -158,7 +158,7 @@ def validate_internal_urls end end - @failures.concat(@internal_link_checks.issues) + @failures.concat(@internal_link_checks.issues) unless @internal_urls.length.zero? end def files From 09b189969bf48f7cc15b4de80b6faeb943354ef6 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 14 Nov 2020 16:20:52 +0000 Subject: [PATCH 006/289] stupid ordering stuff --- lib/html-proofer/runner.rb | 2 +- spec/html-proofer/links_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index df69d59a..8c33f756 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -149,7 +149,7 @@ def validate_internal_urls urls_to_check.each_pair do |url, internal_urls| result = @internal_link_checks.check_internal_link(internal_urls.first.link, internal_urls.first.line, internal_urls.first.content) code = result ? 200 : 404 - @cache.add(url, @internal_urls_to_paths[url], code, '') # TODO: blank msg for now + @cache.add(url, @internal_urls_to_paths[url].sort, code, '') # TODO: blank msg for now end @cache.write else diff --git a/spec/html-proofer/links_spec.rb b/spec/html-proofer/links_spec.rb index cd43120d..aae8ed43 100644 --- a/spec/html-proofer/links_spec.rb +++ b/spec/html-proofer/links_spec.rb @@ -43,7 +43,7 @@ it 'fails for broken internal hash' do broken_hash_internal_filepath = "#{FIXTURES_DIR}/links/broken_hash_internal.html" proofer = run_proofer(broken_hash_internal_filepath, :file) - expect(proofer.failed_tests.first).to match(/linking to internal hash #noHash that does not exist/) + expect(proofer.failed_tests.last).to match(/linking to internal hash #noHash that does not exist/) end it 'passes when linking to the top' do From 4cf141f9231cb78e662741917de4d19da88af059 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 14 Nov 2020 16:13:55 +0000 Subject: [PATCH 007/289] test source tags --- lib/html-proofer/check/links.rb | 6 ++-- spec/html-proofer/fixtures/links/source.html | 6 ++++ ...source_html_log_level_error_type_file_.yml | 28 +++++++++++++++++++ spec/html-proofer/links_spec.rb | 6 ++++ 4 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 spec/html-proofer/fixtures/links/source.html create mode 100644 spec/html-proofer/fixtures/vcr_cassettes/links/source_html_log_level_error_type_file_.yml diff --git a/lib/html-proofer/check/links.rb b/lib/html-proofer/check/links.rb index 02535a29..2b32a65d 100644 --- a/lib/html-proofer/check/links.rb +++ b/lib/html-proofer/check/links.rb @@ -4,7 +4,7 @@ class LinkCheck < ::HTMLProofer::Check include HTMLProofer::Utils def missing_href? - blank?(@link.href) && blank?(@link.name) && blank?(@link.id) + blank?(@link.href) && blank?(@link.name) && blank?(@link.id) && (@node.name == 'source' && blank?(@link.src)) end def placeholder? @@ -12,7 +12,7 @@ def placeholder? end def run - @html.css('a, link').each do |node| + @html.css('a, link, source').each do |node| @link = create_element(node) line = node.line content = node.to_s @@ -49,7 +49,7 @@ def run # curl/Typheous inaccurately return 404s for some links. cc https://git.io/vyCFx next if @link.respond_to?(:rel) && @link.rel == 'dns-prefetch' - add_to_external_urls(@link.href) + add_to_external_urls(@link.href || @link.src) next elsif @link.internal? && !@link.exists? add_issue("internally linking to #{@link.href}, which does not exist", line: line, content: content) diff --git a/spec/html-proofer/fixtures/links/source.html b/spec/html-proofer/fixtures/links/source.html new file mode 100644 index 00000000..5fb350ec --- /dev/null +++ b/spec/html-proofer/fixtures/links/source.html @@ -0,0 +1,6 @@ + + + + diff --git a/spec/html-proofer/fixtures/vcr_cassettes/links/source_html_log_level_error_type_file_.yml b/spec/html-proofer/fixtures/vcr_cassettes/links/source_html_log_level_error_type_file_.yml new file mode 100644 index 00000000..2adfe085 --- /dev/null +++ b/spec/html-proofer/fixtures/vcr_cassettes/links/source_html_log_level_error_type_file_.yml @@ -0,0 +1,28 @@ +--- +http_interactions: +- request: + method: head + uri: http://www.asdo3IRJ395295jsingrkrg4.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.16.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: http://www.asdo3IRJ395295jsingrkrg4.com + recorded_at: Sat, 14 Nov 2020 16:54:52 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html-proofer/links_spec.rb b/spec/html-proofer/links_spec.rb index cd43120d..97dd5e66 100644 --- a/spec/html-proofer/links_spec.rb +++ b/spec/html-proofer/links_spec.rb @@ -656,4 +656,10 @@ proofer = run_proofer(file, :file) expect(proofer.failed_tests).to eq [] end + + it 'checks source tags' do + file = "#{FIXTURES_DIR}/links/source.html" + proofer = run_proofer(file, :file) + expect(proofer.failed_tests.first).to match(/failed/) + end end From 0c205496c979a04811481ec5264cdcfe3fcf8271 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 14 Nov 2020 17:39:26 +0000 Subject: [PATCH 008/289] lint --- .rubocop.yml | 4 ++++ bin/htmlproofer | 2 +- html-proofer.gemspec | 1 + lib/html-proofer/cache.rb | 11 +++++------ lib/html-proofer/check/links.rb | 14 +++++++------- lib/html-proofer/configuration.rb | 2 +- lib/html-proofer/element.rb | 6 ++---- lib/html-proofer/middleware.rb | 1 + lib/html-proofer/url_validator.rb | 26 +++++++++++++------------- lib/html-proofer/utils.rb | 2 +- spec/html-proofer/cache_spec.rb | 6 +++--- spec/html-proofer/element_spec.rb | 2 +- spec/html-proofer/images_spec.rb | 2 +- 13 files changed, 41 insertions(+), 38 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index abdc5b48..cab6b95a 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,3 +1,7 @@ +AllCops: + Exclude: + - "vendor/**/*" + inherit_gem: rubocop-standard: - config/default.yml diff --git a/bin/htmlproofer b/bin/htmlproofer index d971ce7e..a7f42c59 100755 --- a/bin/htmlproofer +++ b/bin/htmlproofer @@ -1,7 +1,7 @@ #!/usr/bin/env ruby # frozen_string_literal: true -STDOUT.sync = true +$stdout.sync = true $LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib') diff --git a/html-proofer.gemspec b/html-proofer.gemspec index af096e48..6c5484d3 100644 --- a/html-proofer.gemspec +++ b/html-proofer.gemspec @@ -17,6 +17,7 @@ Gem::Specification.new do |gem| gem.executables = all_files.grep(%r{^bin/}) { |f| File.basename(f) } gem.test_files = gem.files.grep(%r{^(spec)/}) gem.require_paths = ['lib'] + gem.required_ruby_version = '~> 2.4' gem.add_dependency 'addressable', '~> 2.3' gem.add_dependency 'mercenary', '~> 0.3' diff --git a/lib/html-proofer/cache.rb b/lib/html-proofer/cache.rb index 406fb46b..2e143f79 100644 --- a/lib/html-proofer/cache.rb +++ b/lib/html-proofer/cache.rb @@ -94,12 +94,12 @@ def detect_url_changes(found) del = 0 @cache_log.delete_if do |url, _| url = clean_url(url) - if !found_urls.include?(url) + if found_urls.include?(url) + false + else @logger.log :debug, "Removing #{url} from cache check" del += 1 true - else - false end end @@ -120,9 +120,8 @@ def load? def retrieve_urls(urls) urls_to_check = detect_url_changes(urls) @cache_log.each_pair do |url, cache| - if within_timeframe?(cache['time']) - next if cache['message'].empty? # these were successes to skip - end + next if within_timeframe?(cache['time']) && cache['message'].empty? # these were successes to skip + urls_to_check[url] = cache['filenames'] # recheck expired links end urls_to_check diff --git a/lib/html-proofer/check/links.rb b/lib/html-proofer/check/links.rb index 01d3ef4f..6c89c07b 100644 --- a/lib/html-proofer/check/links.rb +++ b/lib/html-proofer/check/links.rb @@ -52,10 +52,10 @@ def run add_to_external_urls(@link.href || @link.src) next elsif @link.internal? - if !@link.exists? - add_issue("internally linking to #{@link.href}, which does not exist", line: line, content: content) - else + if @link.exists? add_to_internal_urls(@link.href, InternalLink.new(@link, @path, line, content)) + else + add_issue("internally linking to #{@link.href}, which does not exist", line: line, content: content) end end end @@ -102,7 +102,7 @@ def handle_tel(link, line, content) end def handle_hash(link, line, content) - if link.internal? && !hash_exists?(link.html, link.hash) + if link.internal? && !hash_exists?(link.html, link.hash) # rubocop:disable Style/GuardClause return add_issue("linking to internal hash ##{link.hash} that does not exist", line: line, content: content) elsif link.external? return external_link_check(link, line, content) @@ -112,11 +112,11 @@ def handle_hash(link, line, content) end def external_link_check(link, line, content) - if !link.exists? - return add_issue("trying to find hash of #{link.href}, but #{link.absolute_path} does not exist", line: line, content: content) - else + if link.exists? # rubocop:disable Style/GuardClause target_html = create_nokogiri(link.absolute_path) return add_issue("linking to #{link.href}, but #{link.hash} does not exist", line: line, content: content) unless hash_exists?(target_html, link.hash) + else + return add_issue("trying to find hash of #{link.href}, but #{link.absolute_path} does not exist", line: line, content: content) end true diff --git a/lib/html-proofer/configuration.rb b/lib/html-proofer/configuration.rb index 51f63e90..efe2f6fc 100644 --- a/lib/html-proofer/configuration.rb +++ b/lib/html-proofer/configuration.rb @@ -80,7 +80,7 @@ def self.parse_json_option(option_name, config) begin JSON.parse(config) rescue StandardError - raise ArgumentError, "Option '" + option_name + "' did not contain valid JSON." + raise ArgumentError, "Option '#{option_name} did not contain valid JSON." end end end diff --git a/lib/html-proofer/element.rb b/lib/html-proofer/element.rb index 0a8bf836..8b0a6097 100644 --- a/lib/html-proofer/element.rb +++ b/lib/html-proofer/element.rb @@ -108,9 +108,7 @@ def ignore? return true if /^javascript:/.match?(url) # ignore base64 encoded images - if %w[ImageCheck FaviconCheck].include? @type - return true if /^data:image/.match?(url) - end + return true if %w[ImageCheck FaviconCheck].include?(@type) && /^data:image/.match?(url) # ignore user defined URLs return true if ignores_pattern_check(@check.options[:url_ignore]) @@ -187,7 +185,7 @@ def file_path end elsif File.exist?(File.expand_path(path, @check.src)) || File.exist?(File.expand_path(path_dot_ext, @check.src)) # relative links, path is a file base = File.dirname @check.path - elsif File.exist?(File.join(File.dirname(@check.path), path)) || File.exist?(File.join(File.dirname(@check.path), path_dot_ext)) # relative links in nested dir, path is a file + elsif File.exist?(File.join(File.dirname(@check.path), path)) || File.exist?(File.join(File.dirname(@check.path), path_dot_ext)) # rubocop:disable Lint/DuplicateBranch; relative links in nested dir, path is a file base = File.dirname @check.path else # relative link, path is a directory base = @check.path diff --git a/lib/html-proofer/middleware.rb b/lib/html-proofer/middleware.rb index 86966f68..43dd9ee5 100644 --- a/lib/html-proofer/middleware.rb +++ b/lib/html-proofer/middleware.rb @@ -6,6 +6,7 @@ class Middleware class InvalidHtmlError < StandardError def initialize(failures) + super @failures = failures end diff --git a/lib/html-proofer/url_validator.rb b/lib/html-proofer/url_validator.rb index 59e8bf5f..a07a8c3e 100644 --- a/lib/html-proofer/url_validator.rb +++ b/lib/html-proofer/url_validator.rb @@ -43,11 +43,11 @@ def remove_query_values iterable_external_urls = @external_urls.dup @external_urls.each_key do |url| uri = begin - Addressable::URI.parse(url) - rescue URI::Error, Addressable::URI::InvalidURIError - @logger.log :error, "#{url} is an invalid URL" - nil - end + Addressable::URI.parse(url) + rescue URI::Error, Addressable::URI::InvalidURIError + @logger.log :error, "#{url} is an invalid URL" + nil + end next if uri.nil? || uri.query.nil? iterable_external_urls.delete(url) unless new_url_query_values?(uri, paths_with_queries) @@ -102,11 +102,11 @@ def external_link_checker(external_urls) def establish_queue(external_urls) external_urls.each_pair do |url, filenames| url = begin - clean_url(url) - rescue URI::Error, Addressable::URI::InvalidURIError - add_external_issue(filenames, "#{url} is an invalid URL") - next - end + clean_url(url) + rescue URI::Error, Addressable::URI::InvalidURIError + add_external_issue(filenames, "#{url} is an invalid URL") + next + end method = if hash?(url) && @options[:check_external_hash] :get @@ -120,10 +120,10 @@ def establish_queue(external_urls) def clean_url(href) # catch any obvious issues, like strings in port numbers parsed = Addressable::URI.parse(href) - if href !~ /^([!#{$&}-;=?-\[\]_a-z~]|%[0-9a-fA-F]{2})+$/ - parsed.normalize - else + if href =~ /^([!#{$&}-;=?-\[\]_a-z~]|%[0-9a-fA-F]{2})+$/ href + else + parsed.normalize end end diff --git a/lib/html-proofer/utils.rb b/lib/html-proofer/utils.rb index 3733c529..371bcbe8 100644 --- a/lib/html-proofer/utils.rb +++ b/lib/html-proofer/utils.rb @@ -5,7 +5,7 @@ module HTMLProofer module Utils def pluralize(count, single, plural) - "#{count} #{(count == 1 ? single : plural)}" + "#{count} #{count == 1 ? single : plural}" end def create_nokogiri(path) diff --git a/spec/html-proofer/cache_spec.rb b/spec/html-proofer/cache_spec.rb index 9fed2ba0..74e3c31f 100644 --- a/spec/html-proofer/cache_spec.rb +++ b/spec/html-proofer/cache_spec.rb @@ -171,7 +171,7 @@ def read_cache(cache_file) expect_any_instance_of(HTMLProofer::Cache).to receive(:write) root_link = "#{FIXTURES_DIR}/links/root_link/root_link.html" - expect_any_instance_of(HTMLProofer::Cache).to receive(:add).with("/", ["spec/html-proofer/fixtures/links/root_link/root_link.html"], 200, "") + expect_any_instance_of(HTMLProofer::Cache).to receive(:add).with('/', ['spec/html-proofer/fixtures/links/root_link/root_link.html'], 200, '') # we expect one new link to be added because it's within the 30d time frame run_proofer(root_link, :file, disable_external: true, cache: { timeframe: '30d', cache_file: cache_file_name }.merge(default_cache_options)) @@ -186,7 +186,7 @@ def read_cache(cache_file) expect_any_instance_of(HTMLProofer::Cache).to receive(:write) root_link = "#{FIXTURES_DIR}/links/broken_internal_link.html" - expect_any_instance_of(HTMLProofer::Cache).to receive(:add).with("#noHash", ["spec/html-proofer/fixtures/links/broken_internal_link.html"], 404, "") + expect_any_instance_of(HTMLProofer::Cache).to receive(:add).with('#noHash', ['spec/html-proofer/fixtures/links/broken_internal_link.html'], 404, '') # we expect one new link to be added because it's within the 30d time frame run_proofer(root_link, :file, disable_external: true, cache: { timeframe: '30d', cache_file: cache_file_name }.merge(default_cache_options)) @@ -201,7 +201,7 @@ def read_cache(cache_file) expect_any_instance_of(HTMLProofer::Cache).to receive(:write) root_link = "#{FIXTURES_DIR}/links/broken_internal_hashes" - expect_any_instance_of(HTMLProofer::Cache).to receive(:add).with("file.html#noHash", ["spec/html-proofer/fixtures/links/broken_internal_hashes/file1.html", "spec/html-proofer/fixtures/links/broken_internal_hashes/file2.html", "spec/html-proofer/fixtures/links/broken_internal_hashes/file3.html"], 404, "").once + expect_any_instance_of(HTMLProofer::Cache).to receive(:add).with('file.html#noHash', ['spec/html-proofer/fixtures/links/broken_internal_hashes/file1.html', 'spec/html-proofer/fixtures/links/broken_internal_hashes/file2.html', 'spec/html-proofer/fixtures/links/broken_internal_hashes/file3.html'], 404, '').once # we expect one new link to be added because it's within the 30d time frame run_proofer(root_link, :directory, disable_external: true, cache: { timeframe: '30d', cache_file: cache_file_name }.merge(default_cache_options)) diff --git a/spec/html-proofer/element_spec.rb b/spec/html-proofer/element_spec.rb index 51936b13..21e4a878 100644 --- a/spec/html-proofer/element_spec.rb +++ b/spec/html-proofer/element_spec.rb @@ -31,7 +31,7 @@ it 'works for regex patterns' do nokogiri = Nokogiri::HTML5('') checkable = HTMLProofer::Element.new(nokogiri.css('script').first, @check, nil) - expect(checkable.ignores_pattern_check([%r{\/assets\/.*(js|css|png|svg)}])).to eq true + expect(checkable.ignores_pattern_check([%r{/assets/.*(js|css|png|svg)}])).to eq true end it 'works for string patterns' do diff --git a/spec/html-proofer/images_spec.rb b/spec/html-proofer/images_spec.rb index 50511837..92b2b9ef 100644 --- a/spec/html-proofer/images_spec.rb +++ b/spec/html-proofer/images_spec.rb @@ -157,7 +157,7 @@ src_set_check = "#{FIXTURES_DIR}/images/aria_hidden.html" proofer = run_proofer(src_set_check, :file) expect(proofer.failed_tests.size).to eq 1 - expect(proofer.failed_tests.first).to match(%r{image .\/gpl.png does not have an alt attribute}) + expect(proofer.failed_tests.first).to match(%r{image ./gpl.png does not have an alt attribute}) end it 'fails for images with a srcset but missing alt' do From 9052b651a947d44ab724dec714fe77e0d75302d6 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 14 Nov 2020 20:23:36 +0000 Subject: [PATCH 009/289] minor fixes --- lib/html-proofer/check/links.rb | 4 +++- spec/html-proofer/parse_json_option_spec.rb | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/html-proofer/check/links.rb b/lib/html-proofer/check/links.rb index 6c89c07b..86dc5586 100644 --- a/lib/html-proofer/check/links.rb +++ b/lib/html-proofer/check/links.rb @@ -4,7 +4,9 @@ class LinkCheck < ::HTMLProofer::Check include HTMLProofer::Utils def missing_href? - blank?(@link.href) && blank?(@link.name) && blank?(@link.id) && (@node.name == 'source' && blank?(@link.src)) + return blank?(@link.src) if @node.name == 'source' + + blank?(@link.href) && blank?(@link.name) && blank?(@link.id) end def placeholder? diff --git a/spec/html-proofer/parse_json_option_spec.rb b/spec/html-proofer/parse_json_option_spec.rb index fadcb57c..39075cc9 100644 --- a/spec/html-proofer/parse_json_option_spec.rb +++ b/spec/html-proofer/parse_json_option_spec.rb @@ -42,6 +42,6 @@ end it 'Throws an error when the json config is not valid json' do - expect { HTMLProofer::Configuration.parse_json_option('testName', 'abc') }.to raise_error(ArgumentError, "Option 'testName' did not contain valid JSON.") + expect { HTMLProofer::Configuration.parse_json_option('testName', 'abc') }.to raise_error(ArgumentError) end end From 5badc4e90872dd1575db5f3d668d0c256bf04896 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 14 Nov 2020 20:39:46 +0000 Subject: [PATCH 010/289] :gem: bump to 3.17.0 --- lib/html-proofer/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/version.rb b/lib/html-proofer/version.rb index 886139dd..c460ec81 100644 --- a/lib/html-proofer/version.rb +++ b/lib/html-proofer/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HTMLProofer - VERSION = '3.16.0' + VERSION = '3.17.0' end From ffb6bd93ff25a59beec28499b1042dd1ee573e98 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sun, 22 Nov 2020 17:36:53 +0000 Subject: [PATCH 011/289] Corrections to root_dir docs --- README.md | 2 +- bin/htmlproofer | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 33be1b94..e18b6629 100644 --- a/README.md +++ b/README.md @@ -298,7 +298,7 @@ The `HTMLProofer` constructor takes an optional hash of additional options: | `internal_domains`| An array of Strings containing domains that will be treated as internal urls. | `[]` | | `log_level` | Sets the logging level, as determined by [Yell](https://github.com/rudionrails/yell). One of `:debug`, `:info`, `:warn`, `:error`, or `:fatal`. | `:info` | `only_4xx` | Only reports errors for links that fall within the 4xx status code range. | `false` | -| `root_dir` | The absolute path to the directory serving your html-files. Used when running html-proofer on a file, rather than a directory. | "" | +| `root_dir` | The absolute path to the directory serving your html-files. | "" | | `typhoeus_config` | A JSON-formatted string. Parsed using `JSON.parse` and mapped on top of the default configuration values so that they can be overridden. | `{}` | | `url_ignore` | An array of Strings or RegExps containing URLs that are safe to ignore. It affects all HTML attributes. Note that non-HTTP(S) URIs are always ignored. | `[]` | | `url_swap` | A hash containing key-value pairs of `RegExp => String`. It transforms URLs that match `RegExp` into `String` via `gsub`. | `{}` | diff --git a/bin/htmlproofer b/bin/htmlproofer index a7f42c59..8c4c3362 100755 --- a/bin/htmlproofer +++ b/bin/htmlproofer @@ -50,7 +50,7 @@ Mercenary.program(:htmlproofer) do |p| p.option 'typhoeus_config', '--typhoeus-config CONFIG', String, 'JSON-formatted string of Typhoeus config. Will override the html-proofer defaults.' p.option 'url_ignore', '--url-ignore link1,[link2,...]', Array, 'A comma-separated list of Strings or RegExps containing URLs that are safe to ignore. It affects all HTML attributes. Note that non-HTTP(S) URIs are always ignored' p.option 'url_swap', '--url-swap re:string,[re:string,...]', Array, 'A comma-separated list containing key-value pairs of `RegExp => String`. It transforms URLs that match `RegExp` into `String` via `gsub`. The escape sequences `\\:` should be used to produce literal `:`s.' - p.option 'root_dir', '--root-folder PATH', String, 'The absolute path to the directory serving your html-files. Used when running html-proofer on a file, rather than a directory.' + p.option 'root_dir', '--root-dir PATH', String, 'The absolute path to the directory serving your html-files.' p.action do |args, opts| args = ['.'] if args.empty? From fb71f7845471bad68a3ec80ad531b3011cff73c5 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sun, 22 Nov 2020 17:37:01 +0000 Subject: [PATCH 012/289] Let root_dir work for directories --- lib/html-proofer/element.rb | 24 ++++++++----------- spec/html-proofer/command_spec.rb | 6 +++++ .../root_link.html | 1 + .../links/root_folder/other-project/a.html | 0 4 files changed, 17 insertions(+), 14 deletions(-) create mode 100644 spec/html-proofer/fixtures/links/root_folder/documentation-from-my-project/root_link.html create mode 100644 spec/html-proofer/fixtures/links/root_folder/other-project/a.html diff --git a/lib/html-proofer/element.rb b/lib/html-proofer/element.rb index 8b0a6097..71d02dec 100644 --- a/lib/html-proofer/element.rb +++ b/lib/html-proofer/element.rb @@ -176,22 +176,17 @@ def file_path path_dot_ext = path + @check.options[:extension] if @check.options[:assume_extension] - if %r{^/}.match?(path) # path relative to root - if File.directory?(@check.src) - base = @check.src - else - root_dir = @check.options[:root_dir] - base = root_dir || File.dirname(@check.src) - end + if %r{^/}.match?(path) # rubocop:disable Style/ConditionalAssignment path relative to root + base = @check.options[:root_dir] || File.dirname(@check.src) + ap base elsif File.exist?(File.expand_path(path, @check.src)) || File.exist?(File.expand_path(path_dot_ext, @check.src)) # relative links, path is a file - base = File.dirname @check.path + base = File.dirname(@check.path) elsif File.exist?(File.join(File.dirname(@check.path), path)) || File.exist?(File.join(File.dirname(@check.path), path_dot_ext)) # rubocop:disable Lint/DuplicateBranch; relative links in nested dir, path is a file - base = File.dirname @check.path + base = File.dirname(@check.path) else # relative link, path is a directory base = @check.path end - - file = File.join base, path + file = File.join(base, path) if @check.options[:assume_extension] && File.file?("#{file}#{@check.options[:extension]}") file = "#{file}#{@check.options[:extension]}" elsif File.directory?(file) && !unslashed_directory?(file) # implicit index support @@ -203,14 +198,15 @@ def file_path # checks if a file exists relative to the current pwd def exists? - return @checked_paths[absolute_path] if @checked_paths.key? absolute_path + return @checked_paths[absolute_path] if @checked_paths.key?(absolute_path) - @checked_paths[absolute_path] = File.exist? absolute_path + @checked_paths[absolute_path] = File.exist?(absolute_path) end def absolute_path path = file_path || @check.path - File.expand_path path, Dir.pwd + + File.expand_path(path, Dir.pwd) end def ignores_pattern_check(links) diff --git a/spec/html-proofer/command_spec.rb b/spec/html-proofer/command_spec.rb index 34e397ce..8c11ef87 100644 --- a/spec/html-proofer/command_spec.rb +++ b/spec/html-proofer/command_spec.rb @@ -106,6 +106,12 @@ expect(output).to match('successfully') end + it 'navigates above itself in a subdirectory' do + real_link = "#{FIXTURES_DIR}/links/root_folder/documentation-from-my-project/" + output = make_bin("--root-dir #{FIXTURES_DIR}/links/root_folder/", real_link) + expect(output).to match('successfully') + end + it 'has every option for proofer defaults' do match_command_help(HTMLProofer::Configuration::PROOFER_DEFAULTS) end diff --git a/spec/html-proofer/fixtures/links/root_folder/documentation-from-my-project/root_link.html b/spec/html-proofer/fixtures/links/root_folder/documentation-from-my-project/root_link.html new file mode 100644 index 00000000..5f801735 --- /dev/null +++ b/spec/html-proofer/fixtures/links/root_folder/documentation-from-my-project/root_link.html @@ -0,0 +1 @@ + diff --git a/spec/html-proofer/fixtures/links/root_folder/other-project/a.html b/spec/html-proofer/fixtures/links/root_folder/other-project/a.html new file mode 100644 index 00000000..e69de29b From 26c97fa136b7de8cf4776133446144c193b3009d Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sun, 22 Nov 2020 19:14:02 +0000 Subject: [PATCH 013/289] Rubocop --- lib/html-proofer/element.rb | 1 - lib/html-proofer/runner.rb | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/html-proofer/element.rb b/lib/html-proofer/element.rb index 71d02dec..34ca85e6 100644 --- a/lib/html-proofer/element.rb +++ b/lib/html-proofer/element.rb @@ -178,7 +178,6 @@ def file_path if %r{^/}.match?(path) # rubocop:disable Style/ConditionalAssignment path relative to root base = @check.options[:root_dir] || File.dirname(@check.src) - ap base elsif File.exist?(File.expand_path(path, @check.src)) || File.exist?(File.expand_path(path_dot_ext, @check.src)) # relative links, path is a file base = File.dirname(@check.path) elsif File.exist?(File.join(File.dirname(@check.path), path)) || File.exist?(File.join(File.dirname(@check.path), path_dot_ext)) # rubocop:disable Lint/DuplicateBranch; relative links in nested dir, path is a file diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index 8c33f756..d7a65969 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -226,7 +226,7 @@ def print_failed_tests # @return [ Array ] All before_request blocks. def before_request(&block) @before_request ||= [] - @before_request << block if block_given? + @before_request << block if block @before_request end From a016f01f29b3f01421497e7fbc5e11859720f2e2 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sun, 22 Nov 2020 20:51:35 +0000 Subject: [PATCH 014/289] drop the regex --- lib/html-proofer/element.rb | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/lib/html-proofer/element.rb b/lib/html-proofer/element.rb index 34ca85e6..a39068d7 100644 --- a/lib/html-proofer/element.rb +++ b/lib/html-proofer/element.rb @@ -169,6 +169,10 @@ def param_link url.start_with?('?') end + def absolute_path?(path) + path.start_with?('/') + end + def file_path return if path.nil? || path.empty? @@ -176,15 +180,15 @@ def file_path path_dot_ext = path + @check.options[:extension] if @check.options[:assume_extension] - if %r{^/}.match?(path) # rubocop:disable Style/ConditionalAssignment path relative to root - base = @check.options[:root_dir] || File.dirname(@check.src) - elsif File.exist?(File.expand_path(path, @check.src)) || File.exist?(File.expand_path(path_dot_ext, @check.src)) # relative links, path is a file - base = File.dirname(@check.path) - elsif File.exist?(File.join(File.dirname(@check.path), path)) || File.exist?(File.join(File.dirname(@check.path), path_dot_ext)) # rubocop:disable Lint/DuplicateBranch; relative links in nested dir, path is a file - base = File.dirname(@check.path) - else # relative link, path is a directory - base = @check.path - end + base = if absolute_path?(path) # path relative to root + @check.options[:root_dir] || File.dirname(@check.src) + elsif File.exist?(File.expand_path(path, @check.src)) || File.exist?(File.expand_path(path_dot_ext, @check.src)) # relative links, path is a file + File.dirname(@check.path) + elsif File.exist?(File.join(File.dirname(@check.path), path)) || File.exist?(File.join(File.dirname(@check.path), path_dot_ext)) # rubocop:disable Lint/DuplicateBranch; relative links in nested dir, path is a file + File.dirname(@check.path) + else # relative link, path is a directory + @check.path + end file = File.join(base, path) if @check.options[:assume_extension] && File.file?("#{file}#{@check.options[:extension]}") file = "#{file}#{@check.options[:extension]}" From 9a9552dcaf41914bd1be69f19d3ef3ba4ff4a4ad Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sun, 22 Nov 2020 21:44:54 +0000 Subject: [PATCH 015/289] :gem: bump to 3.17.1 --- lib/html-proofer/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/version.rb b/lib/html-proofer/version.rb index c460ec81..92592604 100644 --- a/lib/html-proofer/version.rb +++ b/lib/html-proofer/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HTMLProofer - VERSION = '3.17.0' + VERSION = '3.17.1' end From 2eda1f4b7e9dec38b32d6dc2fdc7d75c4432ca58 Mon Sep 17 00:00:00 2001 From: Olle Jonsson Date: Mon, 23 Nov 2020 08:21:22 +0100 Subject: [PATCH 016/289] CI: Drop unused Travis directive sudo: false --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d0a0dfda..e036be33 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,6 @@ env: global: - NOKOGIRI_USE_SYSTEM_LIBRARIES=true -sudo: false cache: bundler matrix: From 6636e5e2cd8748d9bcea7d081cc2aee4d46fa4ee Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Mon, 23 Nov 2020 10:56:29 +0000 Subject: [PATCH 017/289] Fix internal link handling for dirs w/o root_dir --- lib/html-proofer/element.rb | 7 ++++++- .../html-proofer/fixtures/links/internals/direct_link.html | 2 ++ spec/html-proofer/fixtures/links/internals/page.html | 0 spec/html-proofer/links_spec.rb | 6 ++++++ 4 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 spec/html-proofer/fixtures/links/internals/direct_link.html create mode 100644 spec/html-proofer/fixtures/links/internals/page.html diff --git a/lib/html-proofer/element.rb b/lib/html-proofer/element.rb index a39068d7..0119cff3 100644 --- a/lib/html-proofer/element.rb +++ b/lib/html-proofer/element.rb @@ -181,7 +181,11 @@ def file_path path_dot_ext = path + @check.options[:extension] if @check.options[:assume_extension] base = if absolute_path?(path) # path relative to root - @check.options[:root_dir] || File.dirname(@check.src) + if File.directory?(@check.src) + @check.options[:root_dir] || @check.src + else + @check.options[:root_dir] || File.dirname(@check.src) + end elsif File.exist?(File.expand_path(path, @check.src)) || File.exist?(File.expand_path(path_dot_ext, @check.src)) # relative links, path is a file File.dirname(@check.path) elsif File.exist?(File.join(File.dirname(@check.path), path)) || File.exist?(File.join(File.dirname(@check.path), path_dot_ext)) # rubocop:disable Lint/DuplicateBranch; relative links in nested dir, path is a file @@ -190,6 +194,7 @@ def file_path @check.path end file = File.join(base, path) + if @check.options[:assume_extension] && File.file?("#{file}#{@check.options[:extension]}") file = "#{file}#{@check.options[:extension]}" elsif File.directory?(file) && !unslashed_directory?(file) # implicit index support diff --git a/spec/html-proofer/fixtures/links/internals/direct_link.html b/spec/html-proofer/fixtures/links/internals/direct_link.html new file mode 100644 index 00000000..82dba5b4 --- /dev/null +++ b/spec/html-proofer/fixtures/links/internals/direct_link.html @@ -0,0 +1,2 @@ + + diff --git a/spec/html-proofer/fixtures/links/internals/page.html b/spec/html-proofer/fixtures/links/internals/page.html new file mode 100644 index 00000000..e69de29b diff --git a/spec/html-proofer/links_spec.rb b/spec/html-proofer/links_spec.rb index 8fb028a8..b426277e 100644 --- a/spec/html-proofer/links_spec.rb +++ b/spec/html-proofer/links_spec.rb @@ -662,4 +662,10 @@ proofer = run_proofer(file, :file) expect(proofer.failed_tests.first).to match(/failed/) end + + it 'works for a direct link through directory' do + file = "#{FIXTURES_DIR}/links/internals" + proofer = run_proofer(file, :directory) + expect(proofer.failed_tests).to eq [] + end end From 15bf0da65c4be6aaf2c20386a544d3e821689d95 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Mon, 23 Nov 2020 10:59:19 +0000 Subject: [PATCH 018/289] explain myself --- lib/html-proofer/element.rb | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/html-proofer/element.rb b/lib/html-proofer/element.rb index 0119cff3..03ad8b65 100644 --- a/lib/html-proofer/element.rb +++ b/lib/html-proofer/element.rb @@ -181,11 +181,8 @@ def file_path path_dot_ext = path + @check.options[:extension] if @check.options[:assume_extension] base = if absolute_path?(path) # path relative to root - if File.directory?(@check.src) - @check.options[:root_dir] || @check.src - else - @check.options[:root_dir] || File.dirname(@check.src) - end + # either overwrite with root_dir; or, if source is directory, use that; or, just get the current file's dirname + @check.options[:root_dir] || (File.directory?(@check.src) ? @check.src : File.dirname(@check.src)) elsif File.exist?(File.expand_path(path, @check.src)) || File.exist?(File.expand_path(path_dot_ext, @check.src)) # relative links, path is a file File.dirname(@check.path) elsif File.exist?(File.join(File.dirname(@check.path), path)) || File.exist?(File.join(File.dirname(@check.path), path_dot_ext)) # rubocop:disable Lint/DuplicateBranch; relative links in nested dir, path is a file @@ -193,6 +190,7 @@ def file_path else # relative link, path is a directory @check.path end + file = File.join(base, path) if @check.options[:assume_extension] && File.file?("#{file}#{@check.options[:extension]}") From c7f87fb33e68c514e51d2c2fec9c64792bae36f1 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Mon, 23 Nov 2020 11:22:29 +0000 Subject: [PATCH 019/289] Try adding CI --- .github/workflows/ci.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..6ce15c53 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: Ruby CI + +on: + push: + branches: [$default-branch] + pull_request: + branches: [$default-branch] + +jobs: + test: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + ruby-version: [2.7.x, 2.6.x, 2.5.x, 2.4.x] + + steps: + - uses: actions/checkout@v2 + - name: Set up Ruby ${{ matrix.ruby-version }} + # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby, + # change this to (see https://github.com/ruby/setup-ruby#versioning): + # uses: ruby/setup-ruby@v1 + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true + - name: Install dependencies + run: bundle install + - name: Run tests + run: bundle exec rake From 53066370fa36aaa3abe12997fb850fceb2a9a949 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Mon, 23 Nov 2020 11:23:02 +0000 Subject: [PATCH 020/289] Bye Travis --- .travis.yml | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e036be33..00000000 --- a/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -language: ruby -rvm: - - 2.4.3 - - 2.5.0 - - 2.6.0 - - 2.7.0 - - ruby-head - -git: - depth: 10 - -env: - global: - - NOKOGIRI_USE_SYSTEM_LIBRARIES=true - -cache: bundler - -matrix: - allow_failures: - - rvm: ruby-head - include: - - script: bundle exec rake rubocop - rvm: 2.7.0 From 9cbbcebc2999e66bf06f97f7a45ef68ddf7d47a9 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Mon, 23 Nov 2020 11:27:29 +0000 Subject: [PATCH 021/289] Try adding GitHub Actions lint --- .github/workflows/lint.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..2424ca98 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,20 @@ +name: Linting + +on: + push: + branches: [$default-branch] + pull_request: + branches: [$default-branch] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: 2.7 + bundler-cache: true + - run: bundle install + - name: Rubocop + run: bundle exec rake rubocop From c05791c78bce02f7d5c008dad9c602eb5d2f606c Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Mon, 23 Nov 2020 11:36:51 +0000 Subject: [PATCH 022/289] like this? --- .github/workflows/ci.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ce15c53..86d72927 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,6 @@ name: Ruby CI -on: - push: - branches: [$default-branch] - pull_request: - branches: [$default-branch] +on: [push, pull_request] jobs: test: From a77471424152764c7db4d920e0278611c0270e25 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Mon, 23 Nov 2020 11:38:16 +0000 Subject: [PATCH 023/289] I guess these are all suggestions --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86d72927..467fa9a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: [2.7.x, 2.6.x, 2.5.x, 2.4.x] + ruby-version: [2.7.2, 2.6.6, 2.5.8, 2.4.10] steps: - uses: actions/checkout@v2 From b47c5ffb20412a7317c9a4da2c61698355d177ce Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Mon, 23 Nov 2020 11:42:07 +0000 Subject: [PATCH 024/289] over it --- README.md | 3 --- Rakefile | 2 +- html-proofer.gemspec | 1 - spec/spec_helper.rb | 6 ------ 4 files changed, 1 insertion(+), 11 deletions(-) diff --git a/README.md b/README.md index e18b6629..4d475f63 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,6 @@ If you generate HTML files, _then this tool might be for you_. -[![Build Status](https://travis-ci.org/gjtorikian/html-proofer.svg?branch=master)](https://travis-ci.org/gjtorikian/html-proofer) [![Gem Version](https://badge.fury.io/rb/html-proofer.svg)](http://badge.fury.io/rb/html-proofer) [![codecov](https://codecov.io/gh/gjtorikian/html-proofer/branch/master/graph/badge.svg)](https://codecov.io/gh/gjtorikian/html-proofer) - - ## Project scope HTMLProofer is a set of tests to validate your HTML output. These tests check if your image references are legitimate, if they have alt tags, if your internal links are working, and so on. It's intended to be an all-in-one checker for your output. diff --git a/Rakefile b/Rakefile index d2ce39a4..b5fdc5ac 100644 --- a/Rakefile +++ b/Rakefile @@ -30,7 +30,7 @@ task :proof_readme do File.write('out/README.html', html) opts = { - url_ignore: [/badge.fury.io/, /codecov.io/] + url_ignore: [/badge.fury.io/] } HTMLProofer.check_directory('./out', opts).run end diff --git a/html-proofer.gemspec b/html-proofer.gemspec index 6c5484d3..b0e2c8b0 100644 --- a/html-proofer.gemspec +++ b/html-proofer.gemspec @@ -28,7 +28,6 @@ Gem::Specification.new do |gem| gem.add_dependency 'yell', '~> 2.0' gem.add_development_dependency 'awesome_print' - gem.add_development_dependency 'codecov' gem.add_development_dependency 'pry-byebug' gem.add_development_dependency 'rake' gem.add_development_dependency 'redcarpet' diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index dd63e917..2710ee44 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,11 +1,5 @@ # frozen_string_literal: true -require 'simplecov' -SimpleCov.start -if ENV['CI'] == 'true' - require 'codecov' - SimpleCov.formatter = SimpleCov::Formatter::Codecov -end require 'bundler/setup' require 'vcr' require 'timecop' From 3192c9661d1895d9c5f057e559da1f581fd21a25 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Mon, 23 Nov 2020 11:45:27 +0000 Subject: [PATCH 025/289] :gem: bump to 3.17.2 --- lib/html-proofer/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/version.rb b/lib/html-proofer/version.rb index 92592604..94ad5dc3 100644 --- a/lib/html-proofer/version.rb +++ b/lib/html-proofer/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HTMLProofer - VERSION = '3.17.1' + VERSION = '3.17.2' end From a4638d26e6db523085cd52b6f6e2864040ec4d64 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Mon, 23 Nov 2020 11:51:30 +0000 Subject: [PATCH 026/289] gem updates --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 467fa9a7..f2308b1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,11 @@ name: Ruby CI -on: [push, pull_request] +on: + pull_request: + + push: + branches: + - main jobs: test: From e19f314fceb271ec17bfbf34f311cad4a49fca53 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Mon, 23 Nov 2020 11:55:23 +0000 Subject: [PATCH 027/289] workflow updates --- .github/workflows/ci.yml | 9 +++++---- .github/workflows/lint.yml | 7 ++++--- lib/html-proofer/check/favicon.rb | 2 +- lib/html-proofer/middleware.rb | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2308b1a..4f56e8b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,13 +2,15 @@ name: Ruby CI on: pull_request: - + types: [opened, synchronize, reopened] push: branches: - main jobs: test: + env: + NOKOGIRI_USE_SYSTEM_LIBRARIES: true runs-on: ubuntu-latest strategy: @@ -19,13 +21,12 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Ruby ${{ matrix.ruby-version }} - # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby, - # change this to (see https://github.com/ruby/setup-ruby#versioning): - # uses: ruby/setup-ruby@v1 uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby-version }} bundler-cache: true + - name: Install libxslt and libxml2 + run: sudo apt-get install libxslt-dev libxml2-dev - name: Install dependencies run: bundle install - name: Run tests diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2424ca98..b5953b2e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,10 +1,11 @@ name: Linting on: - push: - branches: [$default-branch] pull_request: - branches: [$default-branch] + types: [opened, synchronize, reopened] + push: + branches: + - main jobs: test: diff --git a/lib/html-proofer/check/favicon.rb b/lib/html-proofer/check/favicon.rb index e681a8a9..71e879cd 100644 --- a/lib/html-proofer/check/favicon.rb +++ b/lib/html-proofer/check/favicon.rb @@ -7,7 +7,7 @@ def run favicon = create_element(node) next if favicon.ignore? - found = true if favicon.rel.split(' ').last.eql? 'icon' + found = true if favicon.rel.split.last.eql? 'icon' break if found end diff --git a/lib/html-proofer/middleware.rb b/lib/html-proofer/middleware.rb index 43dd9ee5..3c2af938 100644 --- a/lib/html-proofer/middleware.rb +++ b/lib/html-proofer/middleware.rb @@ -60,7 +60,7 @@ def call(env) body = [] result.last.each { |e| body << e } - body = body.join('') + body = body.join begin html = body.lstrip rescue StandardError From ea5eea2bacd589e960800d71abd811e46c3391c1 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Mon, 23 Nov 2020 13:38:42 +0000 Subject: [PATCH 028/289] bust cache I guess --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f56e8b9..82d85d34 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby-version }} - bundler-cache: true + # bundler-cache: true - name: Install libxslt and libxml2 run: sudo apt-get install libxslt-dev libxml2-dev - name: Install dependencies From ebd9d4c4dfd610687486f2c2d95c83eeb47ace03 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 28 Nov 2020 20:29:54 +0000 Subject: [PATCH 029/289] no need --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 39f54dc3..9ddffcc5 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,7 @@ docs/ # YARD artifacts .yardoc _yardoc -doc/ + .DS_Store .idea .byebug_history From 93d423532e47ce8bce7e12ba3c2d298dc9c504f2 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 28 Nov 2020 20:31:45 +0000 Subject: [PATCH 030/289] Provide path to issue creation --- lib/html-proofer/check.rb | 4 ++-- lib/html-proofer/check/links.rb | 10 +++++----- lib/html-proofer/runner.rb | 4 ++-- .../fixtures/links/hashes/doc/hcl/index.html | 1 + .../fixtures/links/hashes/downloads/index.html | 1 + .../fixtures/links/hashes/support/index.html | 1 + spec/html-proofer/links_spec.rb | 6 ++++++ 7 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 spec/html-proofer/fixtures/links/hashes/doc/hcl/index.html create mode 100644 spec/html-proofer/fixtures/links/hashes/downloads/index.html create mode 100644 spec/html-proofer/fixtures/links/hashes/support/index.html diff --git a/lib/html-proofer/check.rb b/lib/html-proofer/check.rb index 6ab8c17a..8eeb5018 100644 --- a/lib/html-proofer/check.rb +++ b/lib/html-proofer/check.rb @@ -26,8 +26,8 @@ def run raise NotImplementedError, 'HTMLProofer::Check subclasses must implement #run' end - def add_issue(desc, line: nil, status: -1, content: nil) - @issues << Issue.new(@path, desc, line: line, status: status, content: content) + def add_issue(desc, line: nil, path: nil, status: -1, content: nil) + @issues << Issue.new(path || @path, desc, line: line, status: status, content: content) false end diff --git a/lib/html-proofer/check/links.rb b/lib/html-proofer/check/links.rb index 86dc5586..180e0ef6 100644 --- a/lib/html-proofer/check/links.rb +++ b/lib/html-proofer/check/links.rb @@ -65,15 +65,15 @@ def run external_urls end - def check_internal_link(link, line, content) + def check_internal_link(link, path, line, content) # does the local directory have a trailing slash? if link.unslashed_directory?(link.absolute_path) - add_issue("internally linking to a directory #{link.absolute_path} without trailing slash", line: line, content: content) + add_issue("internally linking to a directory #{link.absolute_path} without trailing slash", path: path, line: line, content: content) return false end # verify the target hash - return handle_hash(link, line, content) if link.hash + return handle_hash(link, path, line, content) if link.hash true end @@ -103,9 +103,9 @@ def handle_tel(link, line, content) add_issue("#{link.href} contains no phone number", line: line, content: content) if link.path.empty? end - def handle_hash(link, line, content) + def handle_hash(link, path, line, content) if link.internal? && !hash_exists?(link.html, link.hash) # rubocop:disable Style/GuardClause - return add_issue("linking to internal hash ##{link.hash} that does not exist", line: line, content: content) + return add_issue("linking to internal hash ##{link.hash} that does not exist", path:path, line: line, content: content) elsif link.external? return external_link_check(link, line, content) end diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index d7a65969..94840a34 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -147,14 +147,14 @@ def validate_internal_urls urls_to_check = load_internal_cache urls_to_check.each_pair do |url, internal_urls| - result = @internal_link_checks.check_internal_link(internal_urls.first.link, internal_urls.first.line, internal_urls.first.content) + result = @internal_link_checks.check_internal_link(internal_urls.first.link, internal_urls.first.path, internal_urls.first.line, internal_urls.first.content) code = result ? 200 : 404 @cache.add(url, @internal_urls_to_paths[url].sort, code, '') # TODO: blank msg for now end @cache.write else @internal_urls.values.flatten.each do |internal_url| - @internal_link_checks.check_internal_link(internal_url.link, internal_url.line, internal_url.content) + @internal_link_checks.check_internal_link(internal_url.link, internal_url.path, internal_url.line, internal_url.content) end end diff --git a/spec/html-proofer/fixtures/links/hashes/doc/hcl/index.html b/spec/html-proofer/fixtures/links/hashes/doc/hcl/index.html new file mode 100644 index 00000000..bf3ae0fb --- /dev/null +++ b/spec/html-proofer/fixtures/links/hashes/doc/hcl/index.html @@ -0,0 +1 @@ +

Generating and Submitting New Reports

diff --git a/spec/html-proofer/fixtures/links/hashes/downloads/index.html b/spec/html-proofer/fixtures/links/hashes/downloads/index.html new file mode 100644 index 00000000..e7ee9eec --- /dev/null +++ b/spec/html-proofer/fixtures/links/hashes/downloads/index.html @@ -0,0 +1 @@ +Just a file. diff --git a/spec/html-proofer/fixtures/links/hashes/support/index.html b/spec/html-proofer/fixtures/links/hashes/support/index.html new file mode 100644 index 00000000..58ad6ad1 --- /dev/null +++ b/spec/html-proofer/fixtures/links/hashes/support/index.html @@ -0,0 +1 @@ +provide your hardware details is by generating and submitting a Hardware diff --git a/spec/html-proofer/links_spec.rb b/spec/html-proofer/links_spec.rb index b426277e..c5811612 100644 --- a/spec/html-proofer/links_spec.rb +++ b/spec/html-proofer/links_spec.rb @@ -668,4 +668,10 @@ proofer = run_proofer(file, :directory) expect(proofer.failed_tests).to eq [] end + + it 'reports linked internal through directory' do + file = "#{FIXTURES_DIR}/links/hashes" + proofer = run_proofer(file, :directory) + expect(proofer.failed_tests.first).to match(%r{support/index.html}) + end end From d4c5b306fbaa40b8b8e2dc3f003fefd6a28f54a0 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 28 Nov 2020 20:32:41 +0000 Subject: [PATCH 031/289] lint --- lib/html-proofer/check/links.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/check/links.rb b/lib/html-proofer/check/links.rb index 180e0ef6..71092840 100644 --- a/lib/html-proofer/check/links.rb +++ b/lib/html-proofer/check/links.rb @@ -105,7 +105,7 @@ def handle_tel(link, line, content) def handle_hash(link, path, line, content) if link.internal? && !hash_exists?(link.html, link.hash) # rubocop:disable Style/GuardClause - return add_issue("linking to internal hash ##{link.hash} that does not exist", path:path, line: line, content: content) + return add_issue("linking to internal hash ##{link.hash} that does not exist", path: path, line: line, content: content) elsif link.external? return external_link_check(link, line, content) end From 21b0edd27c792b8ff1c157189a51fcaeeb92ce86 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 28 Nov 2020 20:35:59 +0000 Subject: [PATCH 032/289] :gem: bump to 3.17.3 --- lib/html-proofer/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/version.rb b/lib/html-proofer/version.rb index 94ad5dc3..0964a499 100644 --- a/lib/html-proofer/version.rb +++ b/lib/html-proofer/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HTMLProofer - VERSION = '3.17.2' + VERSION = '3.17.3' end From ef4eafae95e7aa84e44ae29b9dc448f9953598c9 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 3 Dec 2020 11:17:29 +0000 Subject: [PATCH 033/289] Correct internal links when pulled from outdated cache --- lib/html-proofer/runner.rb | 3 +++ spec/html-proofer/cache_spec.rb | 19 +++++++++++++++++++ .../cache/.not_within_date_internal.log | 1 + 3 files changed, 23 insertions(+) create mode 100644 spec/html-proofer/fixtures/cache/.not_within_date_internal.log diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index 94840a34..d37523c7 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -147,6 +147,9 @@ def validate_internal_urls urls_to_check = load_internal_cache urls_to_check.each_pair do |url, internal_urls| + # pulled from cache + internal_urls = @internal_urls[url] unless internal_urls.first.is_a?(LinkCheck::InternalLink) + result = @internal_link_checks.check_internal_link(internal_urls.first.link, internal_urls.first.path, internal_urls.first.line, internal_urls.first.content) code = result ? 200 : 404 @cache.add(url, @internal_urls_to_paths[url].sort, code, '') # TODO: blank msg for now diff --git a/spec/html-proofer/cache_spec.rb b/spec/html-proofer/cache_spec.rb index 74e3c31f..7c5b97a0 100644 --- a/spec/html-proofer/cache_spec.rb +++ b/spec/html-proofer/cache_spec.rb @@ -144,6 +144,25 @@ def read_cache(cache_file) end end + context 'not within date for internal url' do + let(:cache_file_name) { '.not_within_date_internal.log' } + it 'does write file if timestamp is not within date' do + # this is frozen to within 7 days of the log + new_time = Time.local(2015, 10, 27, 12, 0, 0) + Timecop.freeze(new_time) + + expect_any_instance_of(HTMLProofer::Cache).to receive(:write) + root_link = "#{FIXTURES_DIR}/links/root_link/root_link.html" + + # we expect an add since we are mocking outside the timeframe + expect_any_instance_of(HTMLProofer::Cache).to receive(:add).with('/', ['spec/html-proofer/fixtures/links/root_link/root_link.html'], 200, '') + + run_proofer(root_link, :file, disable_external: true, cache: { timeframe: '4d', cache_file: cache_file_name }.merge(default_cache_options)) + + Timecop.return + end + end + context 'new external url added' do let(:cache_file_name) { '.new_external_url.log' } it 'does write file if a new URL is added' do diff --git a/spec/html-proofer/fixtures/cache/.not_within_date_internal.log b/spec/html-proofer/fixtures/cache/.not_within_date_internal.log new file mode 100644 index 00000000..3353cd02 --- /dev/null +++ b/spec/html-proofer/fixtures/cache/.not_within_date_internal.log @@ -0,0 +1 @@ +{"/":{"time":"2015-10-20 12:00:00 -0700","filenames":["spec/html-proofer/fixtures/links/root_link/root_link.html"],"status":200,"message":""}} From e09f6bedd88e39ca7d641a4358741081b23ce5b6 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 3 Dec 2020 11:22:30 +0000 Subject: [PATCH 034/289] :gem: bump to 3.17.4 --- lib/html-proofer/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/version.rb b/lib/html-proofer/version.rb index 0964a499..b865efa8 100644 --- a/lib/html-proofer/version.rb +++ b/lib/html-proofer/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HTMLProofer - VERSION = '3.17.3' + VERSION = '3.17.4' end From 29d316d434b522713f4365afbec9f1d0dec2d6c3 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 11 Dec 2020 16:50:50 +0000 Subject: [PATCH 035/289] Never needed to raise --- lib/html-proofer/runner.rb | 3 ++- spec/html-proofer/fixtures/links/hash/inner.html | 7 +++++++ spec/spec_helper.rb | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 spec/html-proofer/fixtures/links/hash/inner.html diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index d37523c7..0926ba70 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -214,7 +214,8 @@ def print_failed_tests sorted_failures.sort_and_report count = @failures.length failure_text = pluralize(count, 'failure', 'failures') - raise @logger.colorize :fatal, "HTML-Proofer found #{failure_text}!" + @logger.log :fatal, "HTML-Proofer found #{failure_text}!" + exit 1 end # Set before_request callback. diff --git a/spec/html-proofer/fixtures/links/hash/inner.html b/spec/html-proofer/fixtures/links/hash/inner.html new file mode 100644 index 00000000..8f49f052 --- /dev/null +++ b/spec/html-proofer/fixtures/links/hash/inner.html @@ -0,0 +1,7 @@ +English below + +
+
+ +

English

+ diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 2710ee44..2f3631c3 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -28,7 +28,7 @@ def capture_stderr(*) $stdout = StringIO.new unless ENV['VERBOSE'] begin yield - rescue RuntimeError # rubocop:disable Lint/SuppressedException + rescue SystemExit # rubocop:disable Lint/SuppressedException ensure $stderr = original_stderr $stdout = original_stdout unless ENV['VERBOSE'] From 49ad67527a29078c4f7e4fea94d32274f29583e0 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 11 Dec 2020 16:59:10 +0000 Subject: [PATCH 036/289] Lint --- .rubocop.yml | 14 -------------- lib/html-proofer.rb | 2 ++ lib/html-proofer/url_validator.rb | 2 +- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index cab6b95a..8437a0c5 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,17 +1,3 @@ -AllCops: - Exclude: - - "vendor/**/*" - inherit_gem: rubocop-standard: - config/default.yml - -Style/StringLiterals: - Enabled: true - EnforcedStyle: single_quotes - -Naming/FileName: - Enabled: false - -Style/Documentation: - Enabled: false diff --git a/lib/html-proofer.rb b/lib/html-proofer.rb index d828b453..bd3711d1 100644 --- a/lib/html-proofer.rb +++ b/lib/html-proofer.rb @@ -1,3 +1,4 @@ +# rubocop:disable Naming/FileName # frozen_string_literal: true def require_all(path) @@ -52,3 +53,4 @@ def self.check_links(links, options = {}) HTMLProofer::Runner.new(links, options) end end +# rubocop:enable Naming/FileName diff --git a/lib/html-proofer/url_validator.rb b/lib/html-proofer/url_validator.rb index a07a8c3e..4cf85c11 100644 --- a/lib/html-proofer/url_validator.rb +++ b/lib/html-proofer/url_validator.rb @@ -120,7 +120,7 @@ def establish_queue(external_urls) def clean_url(href) # catch any obvious issues, like strings in port numbers parsed = Addressable::URI.parse(href) - if href =~ /^([!#{$&}-;=?-\[\]_a-z~]|%[0-9a-fA-F]{2})+$/ + if href =~ /^([!#{Regexp.last_match(0)}-;=?-\[\]_a-z~]|%[0-9a-fA-F]{2})+$/ href else parsed.normalize From 5a4afd2c471229a557db3a9af2280c6b32237ecc Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 11 Dec 2020 17:10:55 +0000 Subject: [PATCH 037/289] Install deps first --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82d85d34..1ba668e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,13 +20,13 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Install libxslt and libxml2 + run: sudo apt-get install libxslt-dev libxml2-dev - name: Set up Ruby ${{ matrix.ruby-version }} uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby-version }} # bundler-cache: true - - name: Install libxslt and libxml2 - run: sudo apt-get install libxslt-dev libxml2-dev - name: Install dependencies run: bundle install - name: Run tests From fea77d0179f01679f72944f87388019cacb7f94a Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 11 Dec 2020 17:58:57 +0000 Subject: [PATCH 038/289] Capture errors --- lib/html-proofer/runner.rb | 2 +- spec/html-proofer/proofer_spec.rb | 7 ++++++- spec/spec_helper.rb | 4 +++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index 0926ba70..e853ac3c 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -214,7 +214,7 @@ def print_failed_tests sorted_failures.sort_and_report count = @failures.length failure_text = pluralize(count, 'failure', 'failures') - @logger.log :fatal, "HTML-Proofer found #{failure_text}!" + @logger.log :fatal, "\nHTML-Proofer found #{failure_text}!" exit 1 end diff --git a/spec/html-proofer/proofer_spec.rb b/spec/html-proofer/proofer_spec.rb index bfc4ec52..359aa0ab 100644 --- a/spec/html-proofer/proofer_spec.rb +++ b/spec/html-proofer/proofer_spec.rb @@ -57,6 +57,8 @@ Tel me - spec/html-proofer/fixtures/sorting/path/single_issue.html * image has a terrible filename (./Screen Shot 2012-08-09 at 7.51.18 AM.png) (line 1) + +HTML-Proofer found 4 failures! '.strip) end @@ -71,6 +73,8 @@ * spec/html-proofer/fixtures/sorting/issue/broken_image_two.html (line 1) - internal image NOT_AN_IMAGE does not exist * spec/html-proofer/fixtures/sorting/issue/broken_image_two.html (line 4) + +HTML-Proofer found 5 failures! '.strip) end @@ -82,7 +86,8 @@ - 301 * spec/html-proofer/fixtures/sorting/status/a_404.html: External link http://upload.wikimedia.org/wikipedia/en/thumb/not_here.png failed: 301 * spec/html-proofer/fixtures/sorting/status/broken_link.html: External link http://upload.wikimedia.org/wikipedia/en/thumb/fooooof.png failed: 301 - '.strip) +HTML-Proofer found 3 failures! + '.strip) end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 2f3631c3..9a31b9ab 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -4,6 +4,7 @@ require 'vcr' require 'timecop' require_relative '../lib/html-proofer' +require 'open3' FIXTURES_DIR = 'spec/html-proofer/fixtures' @@ -68,7 +69,8 @@ def send_proofer_output(file, type, opts = {}) end def make_bin(cmd, path = nil) - `bin/htmlproofer #{cmd} #{path}` + stdout, stderr = Open3.capture3("bin/htmlproofer #{cmd} #{path}") + "#{stdout}\n#{stderr}" end def make_cassette_name(file, opts) From fff1bd815c44e63857c0ed8d517afd4c76a9700e Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 11 Dec 2020 18:22:41 +0000 Subject: [PATCH 039/289] 3.18.0 --- lib/html-proofer/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/version.rb b/lib/html-proofer/version.rb index b865efa8..a04f0a4a 100644 --- a/lib/html-proofer/version.rb +++ b/lib/html-proofer/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HTMLProofer - VERSION = '3.17.4' + VERSION = '3.18.0' end From a4417379916fa4eafda8b9eb30dc4c2061ea71b7 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Wed, 16 Dec 2020 19:22:25 +0000 Subject: [PATCH 040/289] Demonstrate broken hash link --- .../fixtures/links/hash/inner.html | 14 +- ...h_href_true_log_level_error_type_file_.yml | 2093 +++++++++++++++++ spec/html-proofer/links_spec.rb | 6 + 3 files changed, 2106 insertions(+), 7 deletions(-) create mode 100644 spec/html-proofer/fixtures/vcr_cassettes/hash/inner_html_allow_hash_href_true_log_level_error_type_file_.yml diff --git a/spec/html-proofer/fixtures/links/hash/inner.html b/spec/html-proofer/fixtures/links/hash/inner.html index 8f49f052..e7a30685 100644 --- a/spec/html-proofer/fixtures/links/hash/inner.html +++ b/spec/html-proofer/fixtures/links/hash/inner.html @@ -1,7 +1,7 @@ -English below - -
-
- -

English

- +Community | Community, a Hugo Theme

Community Resources

Welcome to our community section! Want to add your own information here? You’re very welcome, just go ahead and navigate over to our GitHub repository and follow the instructions to contribute!

diff --git a/spec/html-proofer/fixtures/vcr_cassettes/hash/inner_html_allow_hash_href_true_log_level_error_type_file_.yml b/spec/html-proofer/fixtures/vcr_cassettes/hash/inner_html_allow_hash_href_true_log_level_error_type_file_.yml new file mode 100644 index 00000000..0a3f2197 --- /dev/null +++ b/spec/html-proofer/fixtures/vcr_cassettes/hash/inner_html_allow_hash_href_true_log_level_error_type_file_.yml @@ -0,0 +1,2093 @@ +--- +http_interactions: +- request: + method: head + uri: "#blog" + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: "#blog" + recorded_at: Wed, 16 Dec 2020 19:18:50 GMT +- request: + method: head + uri: "#podcast" + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: "#podcast" + recorded_at: Wed, 16 Dec 2020 19:18:50 GMT +- request: + method: head + uri: https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.1/css/all.min.css + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + date: + - Wed, 16 Dec 2020 19:18:51 GMT + content-type: + - text/css; charset=utf-8 + access-control-allow-origin: + - "*" + cache-control: + - public, max-age=30672000 + etag: + - W/"5f7b5b5f-e7d0" + last-modified: + - Mon, 05 Oct 2020 17:43:59 GMT + cf-cdnjs-via: + - cfworker/kv + cross-origin-resource-policy: + - cross-origin + timing-allow-origin: + - "*" + x-content-type-options: + - nosniff + cf-request-id: + - 070e96bdb70000f41fdc8ec000000001 + expect-ct: + - max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct" + cf-cache-status: + - HIT + age: + - '585892' + expires: + - Mon, 06 Dec 2021 19:18:51 GMT + report-to: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report?s=F55cdj7F4DGjAr5klatpNCcOGj%2Fog7xAOX3D7XsX4Dg2DZdQhHjdis4aAkm64Om9Bq4yp2esvtfsFZBpsJgXu6%2F8Vbjf%2FN2HwXUc8KF%2FRoui%2BqUk7A%3D%3D"}],"group":"cf-nel","max_age":604800}' + nel: + - '{"report_to":"cf-nel","max_age":604800}' + strict-transport-security: + - max-age=15780000 + server: + - cloudflare + cf-ray: + - 602ac0a928b0f41f-LHR + alt-svc: + - h3-27=":443"; ma=86400, h3-28=":443"; ma=86400, h3-29=":443"; ma=86400 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.1/css/all.min.css + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + date: + - Wed, 16 Dec 2020 19:18:51 GMT + content-type: + - application/javascript; charset=utf-8 + access-control-allow-origin: + - "*" + cache-control: + - public, max-age=30672000 + etag: + - W/"5eb03fa9-4f71" + last-modified: + - Mon, 04 May 2020 16:15:37 GMT + cf-cdnjs-via: + - cfworker/kv + cross-origin-resource-policy: + - cross-origin + timing-allow-origin: + - "*" + x-content-type-options: + - nosniff + cf-request-id: + - 070e96bdbc00006b8385bf3000000001 + expect-ct: + - max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct" + cf-cache-status: + - HIT + age: + - '1094617' + expires: + - Mon, 06 Dec 2021 19:18:51 GMT + report-to: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report?s=np9Pjan8f5W1RCqqnqCWoWxWFZ8HXtR%2BaCYWdYPm%2BbPsaRriEZ9f5rXhrwNOdONn7xA8oaD7SgU6kwjY6XuNO2iCWirOsJgyleDmS%2FRLdmFQWm%2BbWQ%3D%3D"}],"group":"cf-nel","max_age":604800}' + nel: + - '{"report_to":"cf-nel","max_age":604800}' + strict-transport-security: + - max-age=15780000 + server: + - cloudflare + cf-ray: + - 602ac0a92b336b83-LHR + alt-svc: + - h3-27=":443"; ma=86400, h3-28=":443"; ma=86400, h3-29=":443"; ma=86400 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://code.jquery.com/jquery-3.3.1.slim.min.js + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + date: + - Wed, 16 Dec 2020 19:18:51 GMT + content-length: + - '69917' + content-type: + - application/javascript; charset=utf-8 + last-modified: + - Sat, 20 Jan 2018 17:26:44 GMT + accept-ranges: + - bytes + server: + - nginx + etag: + - W/"5a637bd4-1111d" + cache-control: + - max-age=315360000 + - public + access-control-allow-origin: + - "*" + vary: + - Accept-Encoding + x-hw: + - 1608146331.dop218.lo4.t,1608146331.cds225.lo4.hn,1608146331.cds248.lo4.c + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://code.jquery.com/jquery-3.3.1.slim.min.js + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"1544639651"' + content-length: + - '140936' + content-type: + - text/css; charset=utf-8 + last-modified: + - Wed, 12 Dec 2018 18:34:11 GMT + accept-ranges: + - bytes + x-cache: + - HIT + vary: + - Accept-Encoding + access-control-allow-origin: + - "*" + cross-origin-resource-policy: + - cross-origin + x-content-type-options: + - nosniff + timing-allow-origin: + - "*" + cache-control: + - public, max-age=31536000 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"1544639634"' + content-length: + - '51039' + content-type: + - text/javascript; charset=utf-8 + last-modified: + - Wed, 12 Dec 2018 18:33:54 GMT + accept-ranges: + - bytes + x-cache: + - HIT + vary: + - Accept-Encoding + access-control-allow-origin: + - "*" + cross-origin-resource-policy: + - cross-origin + x-content-type-options: + - nosniff + timing-allow-origin: + - "*" + cache-control: + - public, max-age=31536000 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://www.cloudwithchris.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + content-length: + - '27470' + content-type: + - text/html + content-md5: + - xJIrXts8bNJSI53iF44gjw== + last-modified: + - Wed, 16 Dec 2020 10:01:52 GMT + accept-ranges: + - bytes + etag: + - '"0x8D8A1A99CA035D8"' + x-cache: + - TCP_HIT + server: + - Windows-Azure-Web/1.0 Microsoft-HTTPAPI/2.0 + x-ms-request-id: + - 9a3585d7-901e-0025-6fdc-d3b575000000 + x-ms-version: + - '2018-03-28' + x-azure-ref: + - 0m13aXwAAAADs/nJVjh9+SKnRiZQ6ermpTE9OMjFFREdFMDIxMwAxODNmZDdiMS0yYzAzLTRmMjctYmU4OS01MjNiYTJjYTY0NDM= + date: + - Wed, 16 Dec 2020 19:18:50 GMT + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://www.cloudwithchris.com + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://fonts.googleapis.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 404 + message: '' + headers: + content-type: + - text/html; charset=utf-8 + cache-control: + - no-cache, no-store, max-age=0, must-revalidate + pragma: + - no-cache + expires: + - Mon, 01 Jan 1990 00:00:00 GMT + date: + - Wed, 16 Dec 2020 19:18:51 GMT + content-length: + - '1588' + server: + - ESF + x-xss-protection: + - '0' + x-content-type-options: + - nosniff + alt-svc: + - h3-29=":443"; ma=2592000,h3-T051=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; + ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43" + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://fonts.googleapis.com + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://gohugo.io/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + cache-control: + - public, max-age=0, must-revalidate + content-type: + - text/html; charset=UTF-8 + date: + - Mon, 14 Dec 2020 15:50:10 GMT + etag: + - '"9008765768e6524464fcc87f66cf7787-ssl"' + link: + - "; rel=preload; as=script, ; rel=preload; + as=style" + referrer-policy: + - origin-when-cross-origin + strict-transport-security: + - max-age=31536000 + x-content-type-options: + - nosniff + x-frame-options: + - DENY + x-xss-protection: + - 1; mode=block + age: + - '185321' + content-length: + - '52628' + server: + - Netlify + x-nf-request-id: + - 4cab4bcc-63d8-4e01-a7b1-8ffea2f074be-11023136 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://gohugo.io/ + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: get + uri: https://fonts.googleapis.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 404 + message: '' + headers: + content-type: + - text/html; charset=utf-8 + cache-control: + - no-cache, no-store, max-age=0, must-revalidate + pragma: + - no-cache + expires: + - Mon, 01 Jan 1990 00:00:00 GMT + date: + - Wed, 16 Dec 2020 19:18:51 GMT + server: + - ESF + x-xss-protection: + - '0' + x-content-type-options: + - nosniff + alt-svc: + - h3-29=":443"; ma=2592000,h3-T051=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; + ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43" + accept-ranges: + - none + vary: + - Accept-Encoding + body: + encoding: ASCII-8BIT + string: !binary |- + PGh0bWwgbGFuZz1lbj48bWV0YSBjaGFyc2V0PXV0Zi04PjxtZXRhIG5hbWU9dmlld3BvcnQgY29udGVudD0iaW5pdGlhbC1zY2FsZT0xLCBtaW5pbXVtLXNjYWxlPTEsIHdpZHRoPWRldmljZS13aWR0aCI+PHRpdGxlPkVycm9yIDQwNCAoTm90IEZvdW5kKSEhMTwvdGl0bGU+PHN0eWxlPip7bWFyZ2luOjA7cGFkZGluZzowfWh0bWwsY29kZXtmb250OjE1cHgvMjJweCBhcmlhbCxzYW5zLXNlcmlmfWh0bWx7YmFja2dyb3VuZDojZmZmO2NvbG9yOiMyMjI7cGFkZGluZzoxNXB4fWJvZHl7Y29sb3I6IzIyMjt0ZXh0LWFsaWduOnVuc2V0O21hcmdpbjo3JSBhdXRvIDA7bWF4LXdpZHRoOjM5MHB4O21pbi1oZWlnaHQ6MTgwcHg7cGFkZGluZzozMHB4IDAgMTVweDt9KiA+IGJvZHl7YmFja2dyb3VuZDp1cmwoLy93d3cuZ29vZ2xlLmNvbS9pbWFnZXMvZXJyb3JzL3JvYm90LnBuZykgMTAwJSA1cHggbm8tcmVwZWF0O3BhZGRpbmctcmlnaHQ6MjA1cHh9cHttYXJnaW46MTFweCAwIDIycHg7b3ZlcmZsb3c6aGlkZGVufXByZXt3aGl0ZS1zcGFjZTpwcmUtd3JhcDt9aW5ze2NvbG9yOiM3Nzc7dGV4dC1kZWNvcmF0aW9uOm5vbmV9YSBpbWd7Ym9yZGVyOjB9QG1lZGlhIHNjcmVlbiBhbmQgKG1heC13aWR0aDo3NzJweCl7Ym9keXtiYWNrZ3JvdW5kOm5vbmU7bWFyZ2luLXRvcDowO21heC13aWR0aDpub25lO3BhZGRpbmctcmlnaHQ6MH19I2xvZ297YmFja2dyb3VuZDp1cmwoLy93d3cuZ29vZ2xlLmNvbS9pbWFnZXMvYnJhbmRpbmcvZ29vZ2xlbG9nby8xeC9nb29nbGVsb2dvX2NvbG9yXzE1MHg1NGRwLnBuZykgbm8tcmVwZWF0O21hcmdpbi1sZWZ0Oi01cHh9QG1lZGlhIG9ubHkgc2NyZWVuIGFuZCAobWluLXJlc29sdXRpb246MTkyZHBpKXsjbG9nb3tiYWNrZ3JvdW5kOnVybCgvL3d3dy5nb29nbGUuY29tL2ltYWdlcy9icmFuZGluZy9nb29nbGVsb2dvLzJ4L2dvb2dsZWxvZ29fY29sb3JfMTUweDU0ZHAucG5nKSBuby1yZXBlYXQgMCUgMCUvMTAwJSAxMDAlOy1tb3otYm9yZGVyLWltYWdlOnVybCgvL3d3dy5nb29nbGUuY29tL2ltYWdlcy9icmFuZGluZy9nb29nbGVsb2dvLzJ4L2dvb2dsZWxvZ29fY29sb3JfMTUweDU0ZHAucG5nKSAwfX1AbWVkaWEgb25seSBzY3JlZW4gYW5kICgtd2Via2l0LW1pbi1kZXZpY2UtcGl4ZWwtcmF0aW86Mil7I2xvZ297YmFja2dyb3VuZDp1cmwoLy93d3cuZ29vZ2xlLmNvbS9pbWFnZXMvYnJhbmRpbmcvZ29vZ2xlbG9nby8yeC9nb29nbGVsb2dvX2NvbG9yXzE1MHg1NGRwLnBuZykgbm8tcmVwZWF0Oy13ZWJraXQtYmFja2dyb3VuZC1zaXplOjEwMCUgMTAwJX19I2xvZ297ZGlzcGxheTppbmxpbmUtYmxvY2s7aGVpZ2h0OjU0cHg7d2lkdGg6MTUwcHh9PC9zdHlsZT48ZGl2IGlkPSJhZi1lcnJvci1jb250YWluZXIiPjxhIGhyZWY9Ly93d3cuZ29vZ2xlLmNvbT48c3BhbiBpZD1sb2dvIGFyaWEtbGFiZWw9R29vZ2xlPjwvc3Bhbj48L2E+PHA+PGI+NDA0LjwvYj4gPGlucz5UaGF04oCZcyBhbiBlcnJvci48L2lucz48cD5UaGUgcmVxdWVzdGVkIFVSTCB3YXMgbm90IGZvdW5kIG9uIHRoaXMgc2VydmVyLiA8aW5zPlRoYXTigJlzIGFsbCB3ZSBrbm93LjwvaW5zPjwvZGl2Pg== + http_version: '2' + adapter_metadata: + effective_url: https://fonts.googleapis.com + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://twitch.tv/CloudWithChris + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: OK + headers: + Connection: + - keep-alive + Content-Type: + - text/html + Set-Cookie: + - server_session_id=58243a5a1db346ac8c089ec369555ade; domain=.twitch.tv; path=/; + secure; samesite=none + - unique_id=5lSCVa2T5Duip7LtNOE0rugTeK2vyVxV; expires=Mon, 16 Dec 2030 19:18:51 + GMT; domain=.twitch.tv; path=/; secure; samesite=none + - unique_id_durable=5lSCVa2T5Duip7LtNOE0rugTeK2vyVxV; expires=Mon, 16 Dec 2030 + 19:18:51 GMT; domain=.twitch.tv; path=/; secure; samesite=none; httponly + - twitch.lohp.countryCode=GB; domain=.twitch.tv; expires=Sat, 14 Dec 2030 19:18:51 + GMT; + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - 1; mode=block + X-Content-Type-Options: + - nosniff + Strict-Transport-Security: + - max-age=31536000 + Cache-Control: + - private, max-age=0 + Date: + - Wed, 16 Dec 2020 19:18:51 GMT + Vary: + - Accept-Encoding + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: https://www.twitch.tv/CloudWithChris + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://twitter.com/erjosito + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + cache-control: + - no-cache, no-store, must-revalidate, pre-check=0, post-check=0 + content-security-policy: + - 'connect-src ''self'' blob: https://*.giphy.com https://*.pscp.tv https://*.video.pscp.tv + https://*.twimg.com https://api.twitter.com https://api-stream.twitter.com + https://ads-api.twitter.com https://caps.twitter.com https://media.riffsy.com + https://pay.twitter.com https://sentry.io https://ton.twitter.com https://twitter.com + https://upload.twitter.com https://www.google-analytics.com https://app.link + https://api2.branch.io https://bnc.lt https://vmap.snappytv.com https://vmapstage.snappytv.com + https://vmaprel.snappytv.com https://vmap.grabyo.com https://dhdsnappytv-vh.akamaihd.net + https://pdhdsnappytv-vh.akamaihd.net https://mdhdsnappytv-vh.akamaihd.net + https://mdhdsnappytv-vh.akamaihd.net https://mpdhdsnappytv-vh.akamaihd.net + https://mmdhdsnappytv-vh.akamaihd.net https://mdhdsnappytv-vh.akamaihd.net + https://mpdhdsnappytv-vh.akamaihd.net https://mmdhdsnappytv-vh.akamaihd.net + https://dwo3ckksxlb0v.cloudfront.net ; default-src ''self''; form-action ''self'' + https://twitter.com https://*.twitter.com; font-src ''self'' https://*.twimg.com; + frame-src ''self'' https://twitter.com https://mobile.twitter.com https://pay.twitter.com + https://cards-frame.twitter.com ; img-src ''self'' blob: data: https://*.cdn.twitter.com + https://ton.twitter.com https://*.twimg.com https://analytics.twitter.com + https://cm.g.doubleclick.net https://www.google-analytics.com https://www.periscope.tv + https://www.pscp.tv https://media.riffsy.com https://*.giphy.com https://*.pscp.tv + https://prod-periscope-profile.*.amazonaws.com https://platform-lookaside.fbsbx.com + https://scontent.xx.fbcdn.net https://*.googleusercontent.com; manifest-src + ''self''; media-src ''self'' blob: https://twitter.com https://*.twimg.com + https://*.vine.co https://*.pscp.tv https://*.video.pscp.tv https://*.giphy.com + https://media.riffsy.com https://dhdsnappytv-vh.akamaihd.net https://pdhdsnappytv-vh.akamaihd.net + https://mdhdsnappytv-vh.akamaihd.net https://mdhdsnappytv-vh.akamaihd.net + https://mpdhdsnappytv-vh.akamaihd.net https://mmdhdsnappytv-vh.akamaihd.net + https://mdhdsnappytv-vh.akamaihd.net https://mpdhdsnappytv-vh.akamaihd.net + https://mmdhdsnappytv-vh.akamaihd.net https://dwo3ckksxlb0v.cloudfront.net; + object-src ''none''; script-src ''self'' ''unsafe-inline'' https://*.twimg.com https://www.google-analytics.com + https://twitter.com https://app.link ''nonce-YjA2Y2Y2YzQtZjU0ZS00ZmI0LWFmNDctNDE2MTA4NTFjZGVl''; + style-src ''self'' ''unsafe-inline'' https://*.twimg.com; worker-src ''self'' + blob:; report-uri https://twitter.com/i/csp_report?a=O5RXE%3D%3D%3D&ro=false' + content-type: + - text/html; charset=utf-8 + cross-origin-opener-policy: + - same-origin + date: + - Wed, 16 Dec 2020 19:18:51 GMT + expiry: + - Tue, 31 Mar 1981 05:00:00 GMT + last-modified: + - Wed, 16 Dec 2020 19:18:51 GMT + pragma: + - no-cache + server: + - tsa_f + set-cookie: + - personalization_id="v1_J0l2hpaJnXZlk806JmFx4g=="; Max-Age=63072000; Expires=Fri, + 16 Dec 2022 19:18:51 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None + - guest_id=v1%3A160814633114180087; Max-Age=63072000; Expires=Fri, 16 Dec 2022 + 19:18:51 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None + strict-transport-security: + - max-age=631138519 + x-connection-hash: + - 5093cdfc1f6fe15183a93bd5e5c7a84b + x-content-type-options: + - nosniff + x-frame-options: + - DENY + x-powered-by: + - Express + x-response-time: + - '154' + x-xss-protection: + - '0' + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://twitter.com/erjosito + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://twitter.com/reddobowen + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + cache-control: + - no-cache, no-store, must-revalidate, pre-check=0, post-check=0 + content-security-policy: + - 'connect-src ''self'' blob: https://*.giphy.com https://*.pscp.tv https://*.video.pscp.tv + https://*.twimg.com https://api.twitter.com https://api-stream.twitter.com + https://ads-api.twitter.com https://caps.twitter.com https://media.riffsy.com + https://pay.twitter.com https://sentry.io https://ton.twitter.com https://twitter.com + https://upload.twitter.com https://www.google-analytics.com https://app.link + https://api2.branch.io https://bnc.lt https://vmap.snappytv.com https://vmapstage.snappytv.com + https://vmaprel.snappytv.com https://vmap.grabyo.com https://dhdsnappytv-vh.akamaihd.net + https://pdhdsnappytv-vh.akamaihd.net https://mdhdsnappytv-vh.akamaihd.net + https://mdhdsnappytv-vh.akamaihd.net https://mpdhdsnappytv-vh.akamaihd.net + https://mmdhdsnappytv-vh.akamaihd.net https://mdhdsnappytv-vh.akamaihd.net + https://mpdhdsnappytv-vh.akamaihd.net https://mmdhdsnappytv-vh.akamaihd.net + https://dwo3ckksxlb0v.cloudfront.net ; default-src ''self''; form-action ''self'' + https://twitter.com https://*.twitter.com; font-src ''self'' https://*.twimg.com; + frame-src ''self'' https://twitter.com https://mobile.twitter.com https://pay.twitter.com + https://cards-frame.twitter.com ; img-src ''self'' blob: data: https://*.cdn.twitter.com + https://ton.twitter.com https://*.twimg.com https://analytics.twitter.com + https://cm.g.doubleclick.net https://www.google-analytics.com https://www.periscope.tv + https://www.pscp.tv https://media.riffsy.com https://*.giphy.com https://*.pscp.tv + https://prod-periscope-profile.*.amazonaws.com https://platform-lookaside.fbsbx.com + https://scontent.xx.fbcdn.net https://*.googleusercontent.com; manifest-src + ''self''; media-src ''self'' blob: https://twitter.com https://*.twimg.com + https://*.vine.co https://*.pscp.tv https://*.video.pscp.tv https://*.giphy.com + https://media.riffsy.com https://dhdsnappytv-vh.akamaihd.net https://pdhdsnappytv-vh.akamaihd.net + https://mdhdsnappytv-vh.akamaihd.net https://mdhdsnappytv-vh.akamaihd.net + https://mpdhdsnappytv-vh.akamaihd.net https://mmdhdsnappytv-vh.akamaihd.net + https://mdhdsnappytv-vh.akamaihd.net https://mpdhdsnappytv-vh.akamaihd.net + https://mmdhdsnappytv-vh.akamaihd.net https://dwo3ckksxlb0v.cloudfront.net; + object-src ''none''; script-src ''self'' ''unsafe-inline'' https://*.twimg.com https://www.google-analytics.com + https://twitter.com https://app.link ''nonce-ZWQyOTNlZmYtYWQ2My00ZTUwLWIxODYtMzkwNzg4ZGUyZTI2''; + style-src ''self'' ''unsafe-inline'' https://*.twimg.com; worker-src ''self'' + blob:; report-uri https://twitter.com/i/csp_report?a=O5RXE%3D%3D%3D&ro=false' + content-type: + - text/html; charset=utf-8 + cross-origin-opener-policy: + - same-origin + date: + - Wed, 16 Dec 2020 19:18:51 GMT + expiry: + - Tue, 31 Mar 1981 05:00:00 GMT + last-modified: + - Wed, 16 Dec 2020 19:18:51 GMT + pragma: + - no-cache + server: + - tsa_f + set-cookie: + - personalization_id="v1_U+1rXWlZlNLXb4+TMJb77A=="; Max-Age=63072000; Expires=Fri, + 16 Dec 2022 19:18:51 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None + - guest_id=v1%3A160814633115045618; Max-Age=63072000; Expires=Fri, 16 Dec 2022 + 19:18:51 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None + strict-transport-security: + - max-age=631138519 + x-connection-hash: + - 648346be014cbf0eb338926a14ceb141 + x-content-type-options: + - nosniff + x-frame-options: + - DENY + x-powered-by: + - Express + x-response-time: + - '164' + x-xss-protection: + - '0' + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://twitter.com/reddobowen + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '0' + content-type: + - text/html; charset=UTF-8 + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"7120dbc1b2a55039bb5c76cb8b1b587f-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583051 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '0' + content-type: + - text/html; charset=UTF-8 + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"7120dbc1b2a55039bb5c76cb8b1b587f-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583052 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/ + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://blog.cloudtrooper.net/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + server: + - nginx + date: + - Wed, 16 Dec 2020 19:18:51 GMT + content-type: + - text/html; charset=UTF-8 + strict-transport-security: + - max-age=31536000 + vary: + - Accept-Encoding + - Cookie + x-hacker: + - If you're reading this, you should visit automattic.com/jobs and apply to + join the fun, mention this header. + link: + - "; rel=shortlink" + x-ac: + - 3.lhr _dfw + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://blog.cloudtrooper.net/ + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/community/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '0' + content-type: + - text/html; charset=UTF-8 + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"607ac188e11bc67afc2cb7a6395682a5-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583053 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/community/ + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://www.linkedin.com/in/chrisreddington + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 999 + message: '' + headers: + cache-control: + - no-cache, no-store + pragma: + - no-cache + content-length: + - '1529' + content-type: + - text/html + expires: + - Thu, 01 Jan 1970 00:00:00 GMT + set-cookie: + - trkCode=ripf; Max-Age=5 + - trkInfo=AQGVhCq32EYx1gAAAXZs_aV4q39M2zH-Dv1Y1CFCSxTiXetGrY7uS8IRTRBvsHz3iHjay7lbvWZjL3_-tCRreYvWgRyPFwJi4Y4P87Hn2Cmg6URI52dM3B07xZVllLR0e1Btn2g=; + Max-Age=5 + - rtc=AQHD8VjbuxCCEAAAAXZs_aV4LTv8LbpDDT4dS6unriWR9cxkzzf0pUZliPXa4AsdFVvaeR5o8cSsaRtEskSdB2GEM-DOZJHdq03bswlzYxXIbBchCIKTu3uYYYdtewygizGR8gBITxiHegHFuTP05yBvKU46oEoAJqSApSS26K7yHlFnbOJvI71ec8ocoJ8umickH-4vF3inWygRSHstX7CpQ-wjzgRf1pNnUI8ptua-mu2IHBgWkcKWS7gMQsJidgVi2-Y=; + Max-Age=120; path=/; domain=.linkedin.com + x-li-fabric: + - prod-lor1 + x-li-pop: + - afd-prod-esv5 + x-li-proto: + - http/2 + x-li-uuid: + - qQYPYpFIURZAYG0MfysAAA== + x-msedge-ref: + - 'Ref A: 7C210ACA8A6B4E2DB8304DDF28181C83 Ref B: LON21EDGE1017 Ref C: 2020-12-16T19:18:51Z' + date: + - Wed, 16 Dec 2020 19:18:50 GMT + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://www.linkedin.com/in/chrisreddington + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/community/images/cloud-with-chris.png + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + accept-ranges: + - bytes + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '55575' + content-type: + - image/png + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"05429358ae64e8d2a8d8bd7a71fb629b-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583054 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/community/images/cloud-with-chris.png + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: get + uri: https://www.linkedin.com/in/chrisreddington + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 999 + message: '' + headers: + cache-control: + - no-cache, no-store + pragma: + - no-cache + content-length: + - '1529' + content-type: + - text/html + expires: + - Thu, 01 Jan 1970 00:00:00 GMT + set-cookie: + - trkCode=ripf; Max-Age=5 + - trkInfo=AQGZrN1OmNfGhQAAAXZs_aV4Hwv5Yx5DzOMID4tY2PXaHdg510p3rbXg0ZWeG-wpsUUBQGHcjgtV9vE731ZFqwbeBT4vK0ILfLe0OQJ7u0L58dsahCYhmFabH1J6juMTP0dRqRo=; + Max-Age=5 + - rtc=AQF1Lp4yPIyEpgAAAXZs_aV4IFATVtMi6bn6mohyNtynDFeQeWwPMcDNdfJw0hCY9wxlp4jdOCjh5Zm7LjcisoPqK3N_r3kuzoSb40wZrVpDJK14LTy7os7SpupPx4tL6tvsJFiYZ6ASeAipLjboMKZiR7BqFLH40YzVlYqy45Re0QHzmPUrpk376KtYJjj7pO0yD4HDT8ND-MIgejzmoJbxXATTIwezel-qA_BHCBp9HIbdqjyPqDExfKo4805ijSCKexU=; + Max-Age=120; path=/; domain=.linkedin.com + x-li-fabric: + - prod-lva1 + x-li-pop: + - afd-prod-edc2 + x-li-proto: + - http/2 + x-li-uuid: + - YaRKapFIURaAETVf/SoAAA== + x-msedge-ref: + - 'Ref A: BDBCD302BDE04F78B21890A2FAA5F097 Ref B: LON21EDGE1017 Ref C: 2020-12-16T19:18:51Z' + date: + - Wed, 16 Dec 2020 19:18:50 GMT + body: + encoding: UTF-8 + string: | + + + + http_version: '2' + adapter_metadata: + effective_url: https://www.linkedin.com/in/chrisreddington + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://www.youtube.com/c/CloudWithChris + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + content-type: + - text/html; charset=utf-8 + x-content-type-options: + - nosniff + expires: + - Tue, 27 Apr 1971 19:44:06 GMT + x-frame-options: + - SAMEORIGIN + cache-control: + - no-cache + p3p: + - CP="This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl=en + for more info." + date: + - Wed, 16 Dec 2020 19:18:51 GMT + server: + - YouTube Frontend Proxy + x-xss-protection: + - '0' + alt-svc: + - h3-29=":443"; ma=2592000,h3-T051=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; + ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43" + accept-ranges: + - none + vary: + - Accept-Encoding + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://www.youtube.com/c/CloudWithChris + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/community/images/cloudtrooper.jpg + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + accept-ranges: + - bytes + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '46691' + content-type: + - image/jpeg + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"7c0fd14040dba96edda6af0cabe00b63-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583055 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/community/images/cloudtrooper.jpg + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/community/index.xml + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + accept-ranges: + - bytes + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '1128' + content-type: + - application/xml + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"972546437456375244014ac5f4479595-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583056 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/community/index.xml + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/events/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '0' + content-type: + - text/html; charset=UTF-8 + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"48a7a456caf945d3a6794afbb9fd9394-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583058 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/events/ + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://github.com/chrisreddington/hugo-community/edit/main/exampleSite/content/community/_index.en.md + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 16 Dec 2020 19:18:51 GMT + Content-Type: + - text/html; charset=utf-8 + Server: + - GitHub.com + Status: + - 200 OK + Vary: + - X-PJAX + - Accept-Encoding, Accept, X-Requested-With + - Accept-Encoding + Cache-Control: + - no-store + ETag: + - W/"1752ff971b9c47025e49d90cd3453786" + Set-Cookie: + - _device_id=8a4f0ddb04bc76c5624f0e43f4637f56; path=/; expires=Thu, 16 Dec 2021 + 19:18:51 GMT; secure; HttpOnly; SameSite=Lax + - _octo=GH1.1.103225655.1608146331; domain=.github.com; path=/; expires=Thu, + 16 Dec 2021 19:18:51 GMT; secure; SameSite=Lax + - logged_in=no; domain=.github.com; path=/; expires=Thu, 16 Dec 2021 19:18:51 + GMT; secure; HttpOnly; SameSite=Lax + - _gh_sess=XIGBlyqWUw7xCI1Jm%2BHYaAOSqqGLOArwCqgy%2BMByhLcFoCtTCXpdsQ5woe8LXV1aIqAWSr4QrkjDAstUwp7vaxRMEjdX0jvaEDEZXSF6gPaNmoFb1oAygdH3ZFtC2EL2WF5NYo%2FSyNZB%2BAc3ovF77CBSwSM5AOp8ay0rIz2y6gQ0vsDV1uDwoVeWHLC9NQPmCLeB56U6qp8HjJUnStjNTvuSGtRfFLEsOnJgOTHMAiKRNsRgayeTMepT9HPffRuq1wxYJjLoOgswe8Kl5p3fYw%3D%3D--USd%2BAWnfLtqpsgB%2F--X3v%2BluXYULG4lelDbnxVOQ%3D%3D; + path=/; secure; HttpOnly; SameSite=Lax + Strict-Transport-Security: + - max-age=31536000; includeSubdomains; preload + X-Frame-Options: + - deny + X-Content-Type-Options: + - nosniff + X-XSS-Protection: + - 1; mode=block + Referrer-Policy: + - origin-when-cross-origin, strict-origin-when-cross-origin + Expect-CT: + - max-age=2592000, report-uri="https://api.github.com/_private/browser/errors" + Content-Security-Policy: + - 'default-src ''none''; base-uri ''self''; block-all-mixed-content; connect-src + ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com + github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com + github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com + cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com; font-src + github.githubassets.com; form-action ''self'' github.com gist.github.com; + frame-ancestors ''none''; frame-src render.githubusercontent.com; img-src + ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com + github-cloud.s3.amazonaws.com *.githubusercontent.com; manifest-src ''self''; + media-src ''none''; script-src github.githubassets.com; style-src ''unsafe-inline'' + github.githubassets.com; worker-src github.com/socket-worker-5029ae85.js gist.github.com/socket-worker-5029ae85.js' + X-GitHub-Request-Id: + - DD2D:E023:741B956:A6A1DAB:5FDA5D9B + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: https://github.com/login?return_to=https%3A%2F%2Fgithub.com%2Fchrisreddington%2Fhugo-community%2Fedit%2Fmain%2FexampleSite%2Fcontent%2Fcommunity%2F_index.en.md + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/img/apple-touch-icon.png + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + accept-ranges: + - bytes + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '6258' + content-type: + - image/png + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"2aa591a7c1c9ebfb4c3e4c5f7c05d3ff-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583061 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/img/apple-touch-icon.png + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/sitemap.xml + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + accept-ranges: + - bytes + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '4116' + content-type: + - application/xml + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"225365820c4ef1b85ab5b206fb9766c8-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583066 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/sitemap.xml + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/img/speakers/chris-reddington.jpg + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + accept-ranges: + - bytes + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '515950' + content-type: + - image/jpeg + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"010704911747e539d8554ed110a979f2-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583062 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/img/speakers/chris-reddington.jpg + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/topics/cloud-architecture/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '0' + content-type: + - text/html; charset=UTF-8 + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"e8d0bc8b97e11833fca8605ec55cd2d2-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583069 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/topics/cloud-architecture/ + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/main.min.a6f7931621d274ee4714b4f47c4e375cb3d046e21a9ebdeb529c04b3f37c6319.css + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + accept-ranges: + - bytes + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '4171' + content-type: + - text/css; charset=UTF-8 + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"06976f79bb6771f7937afe3b1c785509-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583063 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/main.min.a6f7931621d274ee4714b4f47c4e375cb3d046e21a9ebdeb529c04b3f37c6319.css + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/manifest.json + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + accept-ranges: + - bytes + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '579' + content-type: + - application/json + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"c818e8b240abf5d5d23fa976e818c165-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583064 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/manifest.json + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/topics/devops/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '0' + content-type: + - text/html; charset=UTF-8 + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"a7f8fdba2a950346e3d1c07414d034d4-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583070 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/topics/devops/ + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/groups/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '0' + content-type: + - text/html; charset=UTF-8 + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"daf49f986671f6d19edcc43908b42b51-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583060 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/groups/ + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/speakers/chris-reddington/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '0' + content-type: + - text/html; charset=UTF-8 + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"7d7256aa522c1e20fedbf478b289d13d-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583068 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/speakers/chris-reddington/ + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://github.com/chrisreddington/hugo-community + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: OK + headers: + date: + - Wed, 16 Dec 2020 19:18:51 GMT + content-type: + - text/html; charset=utf-8 + server: + - GitHub.com + status: + - 200 OK + vary: + - X-PJAX, Accept-Encoding, Accept, X-Requested-With, Accept-Encoding + etag: + - W/"6d2ade90837dcdbc84f0e52b3ea5b5e7" + cache-control: + - max-age=0, private, must-revalidate + strict-transport-security: + - max-age=31536000; includeSubdomains; preload + x-frame-options: + - deny + x-content-type-options: + - nosniff + x-xss-protection: + - 1; mode=block + referrer-policy: + - no-referrer-when-downgrade + expect-ct: + - max-age=2592000, report-uri="https://api.github.com/_private/browser/errors" + content-security-policy: + - 'default-src ''none''; base-uri ''self''; block-all-mixed-content; connect-src + ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com + github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com + github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com + cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com; font-src + github.githubassets.com; form-action ''self'' github.com gist.github.com; + frame-ancestors ''none''; frame-src render.githubusercontent.com; img-src + ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com + github-cloud.s3.amazonaws.com *.githubusercontent.com; manifest-src ''self''; + media-src ''none''; script-src github.githubassets.com; style-src ''unsafe-inline'' + github.githubassets.com; worker-src github.com/socket-worker-5029ae85.js gist.github.com/socket-worker-5029ae85.js' + Set-Cookie: + - _gh_sess=aFbVXUnpReuCSy3fveWpQQ6qYQUUjHbYJjkNGO2JBt2h%2BU5q%2BTrEosJHNBVOsEasTf7KvGYyqQarFtORv3LwKnSRMoU%2FGTA6%2FJF4vid%2FTdrIvIv2R23%2B8pyDFaFyakA4CvOwz0kvO9TafPd7YUErR%2BIs5xa4NvZyRe2760wpXHwCdTxiEO6sNBG7qHF6iEobMXXYT9IhVRA052YOn3CANhj%2ByvnROIB%2FEehkq4BE4yNEjgU32MF18lUccZpQPmTVVr8mhyrVeTgyd8c51irLyQ%3D%3D--PYsrn5xtSzb4SNju--0n7eN3l1lbMVddIm7mZfSw%3D%3D; + Path=/; HttpOnly; Secure; SameSite=Lax + - _octo=GH1.1.1813049470.1608146331; Path=/; Domain=github.com; Expires=Thu, + 16 Dec 2021 19:18:51 GMT; Secure; SameSite=Lax + - logged_in=no; Path=/; Domain=github.com; Expires=Thu, 16 Dec 2021 19:18:51 + GMT; HttpOnly; Secure; SameSite=Lax + Accept-Ranges: + - bytes + X-GitHub-Request-Id: + - DD2B:44C7:56AF893:7C31F74:5FDA5D9A + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: https://github.com/chrisreddington/hugo-community + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://deploy-preview-44--hugo-community.netlify.app/speakers/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + cache-control: + - public, max-age=0, must-revalidate + content-length: + - '0' + content-type: + - text/html; charset=UTF-8 + date: + - Wed, 16 Dec 2020 19:18:51 GMT + etag: + - '"1f66883ba4dbbd562775a92658139c8b-ssl"' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-robots-tag: + - noindex + age: + - '0' + server: + - Netlify + x-nf-request-id: + - 868595f1-3060-429b-9069-d81664e2530d-66583067 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://deploy-preview-44--hugo-community.netlify.app/speakers/ + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://github.com/chrisreddington/hugo-community/commit/d142b3b713255f9b9a31fd08db5031fd6e8389f9 + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: OK + headers: + date: + - Wed, 16 Dec 2020 19:18:51 GMT + content-type: + - text/html; charset=utf-8 + server: + - GitHub.com + status: + - 200 OK + vary: + - X-PJAX, Accept-Encoding, Accept, X-Requested-With, Accept-Encoding + etag: + - W/"2e4e3520fe270568380a06f4cd774e28" + cache-control: + - max-age=0, private, must-revalidate + strict-transport-security: + - max-age=31536000; includeSubdomains; preload + x-frame-options: + - deny + x-content-type-options: + - nosniff + x-xss-protection: + - 1; mode=block + referrer-policy: + - no-referrer-when-downgrade + expect-ct: + - max-age=2592000, report-uri="https://api.github.com/_private/browser/errors" + content-security-policy: + - 'default-src ''none''; base-uri ''self''; block-all-mixed-content; connect-src + ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com + github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com + github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com + cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com; font-src + github.githubassets.com; form-action ''self'' github.com gist.github.com; + frame-ancestors ''none''; frame-src render.githubusercontent.com; img-src + ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com + github-cloud.s3.amazonaws.com *.githubusercontent.com; manifest-src ''self''; + media-src ''none''; script-src github.githubassets.com; style-src ''unsafe-inline'' + github.githubassets.com; worker-src github.com/socket-worker-5029ae85.js gist.github.com/socket-worker-5029ae85.js' + Set-Cookie: + - _gh_sess=qYfdKxylirhFsrDXj8%2FZOzjZk0FVffKUVgIUPMCnr2nPQ5YatkEYe2XAjvKYnKOSGs9ZB8TVNNuvUtZbe%2FdhRpQwXx%2FfTlnFamQdqSHP1sXqIXADY9ufo1fbKbiBCjnNFmEgA4EGAnU1g0o2HwGqBj2J1ea%2BS1Zx%2FA5EFQiLE6dUPbMUdFqMl%2BInDxH9Nc4yhqGwzmQwUpd%2BQ7%2FuAbzDxf5i1C3EZPcPlrpJ2E%2FlzuuvBsaFSj7w33Uv6L5dbrkx8jaJjuI6vUj%2BAao4KIAY9Q%3D%3D--GDryR%2B8bcH%2BbA6cU--1xn9nBEFcdzgPnoXNviUyw%3D%3D; + Path=/; HttpOnly; Secure; SameSite=Lax + - _octo=GH1.1.483072665.1608146331; Path=/; Domain=github.com; Expires=Thu, + 16 Dec 2021 19:18:51 GMT; Secure; SameSite=Lax + - logged_in=no; Path=/; Domain=github.com; Expires=Thu, 16 Dec 2021 19:18:51 + GMT; HttpOnly; Secure; SameSite=Lax + Accept-Ranges: + - bytes + X-GitHub-Request-Id: + - DD2C:6667:56AFB1A:7C50F8A:5FDA5D9B + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: https://github.com/chrisreddington/hugo-community/commit/d142b3b713255f9b9a31fd08db5031fd6e8389f9 + recorded_at: Wed, 16 Dec 2020 19:18:51 GMT +- request: + method: head + uri: https://github.com/chrisreddington + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.0; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: OK + headers: + date: + - Wed, 16 Dec 2020 19:18:52 GMT + content-type: + - text/html; charset=utf-8 + server: + - GitHub.com + status: + - 200 OK + vary: + - X-Requested-With, Accept-Encoding, Accept, X-Requested-With, Accept-Encoding + etag: + - W/"8a362acea5ed90e98773df107e63acec" + cache-control: + - max-age=0, private, must-revalidate + strict-transport-security: + - max-age=31536000; includeSubdomains; preload + x-frame-options: + - deny + x-content-type-options: + - nosniff + x-xss-protection: + - 1; mode=block + referrer-policy: + - origin-when-cross-origin, strict-origin-when-cross-origin + expect-ct: + - max-age=2592000, report-uri="https://api.github.com/_private/browser/errors" + content-security-policy: + - 'default-src ''none''; base-uri ''self''; block-all-mixed-content; connect-src + ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com + github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com + github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com + cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com; font-src + github.githubassets.com; form-action ''self'' github.com gist.github.com; + frame-ancestors ''none''; frame-src render.githubusercontent.com; img-src + ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com + github-cloud.s3.amazonaws.com *.githubusercontent.com; manifest-src ''self''; + media-src ''none''; script-src github.githubassets.com; style-src ''unsafe-inline'' + github.githubassets.com; worker-src github.com/socket-worker-5029ae85.js gist.github.com/socket-worker-5029ae85.js' + Set-Cookie: + - _gh_sess=k2lVkmrOE6PvUI4cvuo979LD1uHivKWDmLb99doE1NiLKdYSY7kxvr3BbpwGrB1V6VugEXc8JXeh18pS2V7i2pID0CLPR2kaYwV%2B5tb7ASDepotE5XtpV%2FFIM%2FkkEb%2FY63lyBVYcceD4OqtPBqeMQN27NBDDzAskgGBPGpgpLLutA62NLPQQhgToLi4pRI1dCR12DKOEaEy5%2Fmu4UBXHESdfMSjIk90nvkSB06Yf4RioEoDaYBcun5Njjde5k5mgqxvvPcOv15enih1tEi%2FQ2A%3D%3D--JeuJ3tlgAQo69meU--8Vrj8ppCX9hpQ6IPUOLhEg%3D%3D; + Path=/; HttpOnly; Secure; SameSite=Lax + - _octo=GH1.1.212825667.1608146331; Path=/; Domain=github.com; Expires=Thu, + 16 Dec 2021 19:18:51 GMT; Secure; SameSite=Lax + - logged_in=no; Path=/; Domain=github.com; Expires=Thu, 16 Dec 2021 19:18:51 + GMT; HttpOnly; Secure; SameSite=Lax + Accept-Ranges: + - bytes + X-GitHub-Request-Id: + - DD2A:A7AB:5E4BBB:7FBD4C:5FDA5D9A + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: https://github.com/chrisreddington + recorded_at: Wed, 16 Dec 2020 19:18:52 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html-proofer/links_spec.rb b/spec/html-proofer/links_spec.rb index c5811612..416cbf00 100644 --- a/spec/html-proofer/links_spec.rb +++ b/spec/html-proofer/links_spec.rb @@ -674,4 +674,10 @@ proofer = run_proofer(file, :directory) expect(proofer.failed_tests.first).to match(%r{support/index.html}) end + + it 'works for hash hrefs' do + file = "#{FIXTURES_DIR}/links/hash/inner.html" + proofer = run_proofer(file, :file, allow_hash_href: true) + expect(proofer.failed_tests).to eq [] + end end From 58368c5f38360df30a64a477aa3ade19bc0679af Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Wed, 16 Dec 2020 19:22:43 +0000 Subject: [PATCH 041/289] Fix internal hash ref --- lib/html-proofer/check/links.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/check/links.rb b/lib/html-proofer/check/links.rb index 71092840..44a79720 100644 --- a/lib/html-proofer/check/links.rb +++ b/lib/html-proofer/check/links.rb @@ -45,7 +45,7 @@ def run # intentionally here because we still want valid? & missing_href? to execute next if @link.non_http_remote? - if !@link.internal? && @link.remote? + if !@link.href.start_with?('#') && !@link.internal? && @link.remote? check_sri(line, content) if @link.check_sri? && node.name == 'link' # we need to skip these for now; although the domain main be valid, # curl/Typheous inaccurately return 404s for some links. cc https://git.io/vyCFx From 7013ddfaad182b536dc7aa6f7fcde022cdbbceb3 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Wed, 16 Dec 2020 19:27:49 +0000 Subject: [PATCH 042/289] Our old friend ampersand --- lib/html-proofer/check/links.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/check/links.rb b/lib/html-proofer/check/links.rb index 44a79720..c295a1e0 100644 --- a/lib/html-proofer/check/links.rb +++ b/lib/html-proofer/check/links.rb @@ -45,7 +45,7 @@ def run # intentionally here because we still want valid? & missing_href? to execute next if @link.non_http_remote? - if !@link.href.start_with?('#') && !@link.internal? && @link.remote? + if !@link.href&.start_with?('#') && !@link.internal? && @link.remote? check_sri(line, content) if @link.check_sri? && node.name == 'link' # we need to skip these for now; although the domain main be valid, # curl/Typheous inaccurately return 404s for some links. cc https://git.io/vyCFx From 797970d623fe166d8faafa5dbf2e0db6d4d9f6e1 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Wed, 16 Dec 2020 19:37:01 +0000 Subject: [PATCH 043/289] :gem: bump to 3.18.1 --- lib/html-proofer/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/version.rb b/lib/html-proofer/version.rb index a04f0a4a..61daa4ed 100644 --- a/lib/html-proofer/version.rb +++ b/lib/html-proofer/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HTMLProofer - VERSION = '3.18.0' + VERSION = '3.18.1' end From 893a7cf1f8a0f2ad258f57414cde20585a49b5a8 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 17 Dec 2020 10:38:48 +0000 Subject: [PATCH 044/289] Demonstrate link problem --- lib/html-proofer/runner.rb | 2 +- .../2020/01/13/Citrix/index.html | 265 ++++++++++++++++++ spec/html-proofer/links_spec.rb | 7 + 3 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 spec/html-proofer/fixtures/links/relative_nested_internal_hash/2020/01/13/Citrix/index.html diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index e853ac3c..dc9371a8 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -161,7 +161,7 @@ def validate_internal_urls end end - @failures.concat(@internal_link_checks.issues) unless @internal_urls.length.zero? + # @failures.concat(@internal_link_checks.issues) unless @internal_urls.length.zero? end def files diff --git a/spec/html-proofer/fixtures/links/relative_nested_internal_hash/2020/01/13/Citrix/index.html b/spec/html-proofer/fixtures/links/relative_nested_internal_hash/2020/01/13/Citrix/index.html new file mode 100644 index 00000000..b15f80c4 --- /dev/null +++ b/spec/html-proofer/fixtures/links/relative_nested_internal_hash/2020/01/13/Citrix/index.html @@ -0,0 +1,265 @@ + + + + + + + + +Wijd verspreide kwetsbaarheid in Citrix Gateway en Citrix Application Delivery Controller | Nederlands Security Meldpunt + + + + + + + + + + + + + + + + + + Skip to the content. + + + + + + + + + + +
+

Nederlands Security Meldpunt

+

Het Nederlandse meldpunt voor kwetsbaarheden, infecties en andere (cyber)security gerelateerde zaken.

+
+ + + + + + +
+
+

Wijd verspreide kwetsbaarheid in Citrix Gateway en Citrix Application Delivery Controller

+

+ 13 Jan 2020 - +
+ English below +

+

Het probleem

+ +

Houdt er rekening mee dat apparaten al voor het uitvoeren van de patch door een aanvaller overgenomen kunnen zijn.

+ +

Update: bevat meer informatie over hoe je kunt controleren of je Citrix ADC is overgenomen.

+ +

Wat wij doen…

+ +

Vanochtend ca 11:00 waren er bij het Security Meldpunt 546 via het publieke internet bereikbare kwetsbare Citrix ADC servers bekend. Wij zijn op dit moment actief bezig de beheerders van de netwerken waar deze servers in staan op de hoogte te stellen.

+ +
+ +

English

+ +

The problem

+ +

Please take note that it is possible that devices that did not have patches or mitigations applied may already have been taken over by an attacker.

+ +

Update: contains information on how to check if/how your Citrix AC is compromised.

+ +

What we are doing.

+ +

As of this morning around 11:00 we were aware of 546 publicly accessible vulnerable devices in The Netherlands. We are currently actively informing the owners of the networks containing vulnerable devices.

+ + +
+ + +
+
+ + +
+
+
+
+
+
+ + + + + diff --git a/spec/html-proofer/links_spec.rb b/spec/html-proofer/links_spec.rb index 416cbf00..f2d18874 100644 --- a/spec/html-proofer/links_spec.rb +++ b/spec/html-proofer/links_spec.rb @@ -680,4 +680,11 @@ proofer = run_proofer(file, :file, allow_hash_href: true) expect(proofer.failed_tests).to eq [] end + + it 'works for relative hash references' do + dir = "#{FIXTURES_DIR}/links/relative_nested_internal_hash" + proofer = run_proofer(dir, :directory, allow_hash_href: true) + + expect(proofer.failed_tests).to eq [] + end end From 4eaa9b23180b0b81e26bb2c88eec87ebaa6d0fa5 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 17 Dec 2020 10:59:56 +0000 Subject: [PATCH 045/289] Hash lookup correction --- lib/html-proofer/check/links.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/html-proofer/check/links.rb b/lib/html-proofer/check/links.rb index c295a1e0..72308b9b 100644 --- a/lib/html-proofer/check/links.rb +++ b/lib/html-proofer/check/links.rb @@ -45,7 +45,7 @@ def run # intentionally here because we still want valid? & missing_href? to execute next if @link.non_http_remote? - if !@link.href&.start_with?('#') && !@link.internal? && @link.remote? + if !@link.hash.nil? && !@link.internal? && @link.remote? check_sri(line, content) if @link.check_sri? && node.name == 'link' # we need to skip these for now; although the domain main be valid, # curl/Typheous inaccurately return 404s for some links. cc https://git.io/vyCFx @@ -54,7 +54,7 @@ def run add_to_external_urls(@link.href || @link.src) next elsif @link.internal? - if @link.exists? + if @link.exists? || @link.hash add_to_internal_urls(@link.href, InternalLink.new(@link, @path, line, content)) else add_issue("internally linking to #{@link.href}, which does not exist", line: line, content: content) @@ -72,10 +72,10 @@ def check_internal_link(link, path, line, content) return false end - # verify the target hash - return handle_hash(link, path, line, content) if link.hash + return true unless link.hash - true + # verify the target hash + handle_hash(link, path, line, content) end def check_schemes(link, line, content) From 59d234440f74659853bdad734cd816a0e88d8ab4 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 17 Dec 2020 11:09:52 +0000 Subject: [PATCH 046/289] Don't work like that --- lib/html-proofer/check/links.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/check/links.rb b/lib/html-proofer/check/links.rb index 72308b9b..3c9322ec 100644 --- a/lib/html-proofer/check/links.rb +++ b/lib/html-proofer/check/links.rb @@ -45,7 +45,7 @@ def run # intentionally here because we still want valid? & missing_href? to execute next if @link.non_http_remote? - if !@link.hash.nil? && !@link.internal? && @link.remote? + if !@link.href&.start_with?('#') && !@link.internal? && @link.remote? check_sri(line, content) if @link.check_sri? && node.name == 'link' # we need to skip these for now; although the domain main be valid, # curl/Typheous inaccurately return 404s for some links. cc https://git.io/vyCFx From 92e9d8c55d16d0a05583ad6d5a877b6d5fbf0786 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 17 Dec 2020 15:17:40 +0000 Subject: [PATCH 047/289] Rework this concat a bit --- lib/html-proofer/runner.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index dc9371a8..2ef29f4c 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -157,11 +157,12 @@ def validate_internal_urls @cache.write else @internal_urls.values.flatten.each do |internal_url| - @internal_link_checks.check_internal_link(internal_url.link, internal_url.path, internal_url.line, internal_url.content) + result = @internal_link_checks.check_internal_link(internal_url.link, internal_url.path, internal_url.line, internal_url.content) + next if result + + @failures.concat(@internal_link_checks.issues) unless @internal_urls.length.zero? end end - - # @failures.concat(@internal_link_checks.issues) unless @internal_urls.length.zero? end def files From a55571466c28b20e2445d32384ced0c7be0d8571 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 17 Dec 2020 16:23:39 +0000 Subject: [PATCH 048/289] Expand the concept of internal --- lib/html-proofer/element.rb | 2 +- lib/html-proofer/runner.rb | 6 +- .../2020/01/13/Citrix/index.html | 324 ++++-------------- spec/html-proofer/links_spec.rb | 5 +- 4 files changed, 79 insertions(+), 258 deletions(-) diff --git a/lib/html-proofer/element.rb b/lib/html-proofer/element.rb index 03ad8b65..9b09ef24 100644 --- a/lib/html-proofer/element.rb +++ b/lib/html-proofer/element.rb @@ -158,7 +158,7 @@ def relative_link? end def link_points_to_same_page? - hash_link || param_link + hash_link || param_link || internal_absolute_link? end def hash_link diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index 2ef29f4c..ec91e709 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -157,12 +157,10 @@ def validate_internal_urls @cache.write else @internal_urls.values.flatten.each do |internal_url| - result = @internal_link_checks.check_internal_link(internal_url.link, internal_url.path, internal_url.line, internal_url.content) - next if result - - @failures.concat(@internal_link_checks.issues) unless @internal_urls.length.zero? + @internal_link_checks.check_internal_link(internal_url.link, internal_url.path, internal_url.line, internal_url.content) end end + @failures.concat(@internal_link_checks.issues) unless @internal_urls.length.zero? end def files diff --git a/spec/html-proofer/fixtures/links/relative_nested_internal_hash/2020/01/13/Citrix/index.html b/spec/html-proofer/fixtures/links/relative_nested_internal_hash/2020/01/13/Citrix/index.html index b15f80c4..291c145f 100644 --- a/spec/html-proofer/fixtures/links/relative_nested_internal_hash/2020/01/13/Citrix/index.html +++ b/spec/html-proofer/fixtures/links/relative_nested_internal_hash/2020/01/13/Citrix/index.html @@ -1,265 +1,87 @@ - + - - -Wijd verspreide kwetsbaarheid in Citrix Gateway en Citrix Application Delivery Controller | Nederlands Security Meldpunt - - - - - - - - - - - - + - - + + Skip to the content. - - - - - - - - - - -
-

Nederlands Security Meldpunt

-

Het Nederlandse meldpunt voor kwetsbaarheden, infecties en andere (cyber)security gerelateerde zaken.

-
- - - - - - -
-
-

Wijd verspreide kwetsbaarheid in Citrix Gateway en Citrix Application Delivery Controller

-

- 13 Jan 2020 - -
- English below -

-

Het probleem

- -

Houdt er rekening mee dat apparaten al voor het uitvoeren van de patch door een aanvaller overgenomen kunnen zijn.

- -

Update: bevat meer informatie over hoe je kunt controleren of je Citrix ADC is overgenomen.

- -

Wat wij doen…

- -

Vanochtend ca 11:00 waren er bij het Security Meldpunt 546 via het publieke internet bereikbare kwetsbare Citrix ADC servers bekend. Wij zijn op dit moment actief bezig de beheerders van de netwerken waar deze servers in staan op de hoogte te stellen.

- -
- -

English

- -

The problem

- -

Please take note that it is possible that devices that did not have patches or mitigations applied may already have been taken over by an attacker.

- -

Update: contains information on how to check if/how your Citrix AC is compromised.

- -

What we are doing.

- -

As of this morning around 11:00 we were aware of 546 publicly accessible vulnerable devices in The Netherlands. We are currently actively informing the owners of the networks containing vulnerable devices.

- - -
- - + + + +
+

Nederlands Security Meldpunt +

+

Het Nederlandse meldpunt voor kwetsbaarheden, infecties en andere (cyber)security gerelateerde zaken. +

-
- - -
-
+ +
+
+

Wijd verspreide kwetsbaarheid in Citrix Gateway en Citrix Application Delivery Controller +

+

+ 13 Jan 2020 - +
+ English below +

+

Het probleem +

+

Houdt er rekening mee dat apparaten al voor het uitvoeren van de patch door een aanvaller overgenomen kunnen zijn. +

+

Update: bevat meer informatie over hoe je kunt controleren of je Citrix ADC is overgenomen. +

+

Wat wij doen… +

+

Vanochtend ca 11:00 waren er bij het Security Meldpunt 546 via het publieke internet bereikbare kwetsbare Citrix ADC servers bekend. Wij zijn op dit moment actief bezig de beheerders van de netwerken waar deze servers in staan op de hoogte te stellen. +

+
+

+ English +

+

The problem +

+

Please take note that it is possible that devices that did not have patches or mitigations applied may already have been taken over by an attacker. +

+

Update: contains information on how to check if/how your Citrix AC is compromised. +

+

What we are doing. +

+

As of this morning around 11:00 we were aware of 546 publicly accessible vulnerable devices in The Netherlands. We are currently actively informing the owners of the networks containing vulnerable devices. +

+
+
+
+
+
-
-
- - - +
+
+ - diff --git a/spec/html-proofer/links_spec.rb b/spec/html-proofer/links_spec.rb index f2d18874..4dda74b6 100644 --- a/spec/html-proofer/links_spec.rb +++ b/spec/html-proofer/links_spec.rb @@ -43,7 +43,8 @@ it 'fails for broken internal hash' do broken_hash_internal_filepath = "#{FIXTURES_DIR}/links/broken_hash_internal.html" proofer = run_proofer(broken_hash_internal_filepath, :file) - expect(proofer.failed_tests.last).to match(/linking to internal hash #noHash that does not exist/) + + expect(proofer.failed_tests.first).to match(/linking to internal hash #noHash that does not exist/) end it 'passes when linking to the top' do @@ -681,7 +682,7 @@ expect(proofer.failed_tests).to eq [] end - it 'works for relative hash references' do + it 'works for hash references in self' do dir = "#{FIXTURES_DIR}/links/relative_nested_internal_hash" proofer = run_proofer(dir, :directory, allow_hash_href: true) From 8f8f0a4c4e74d0e42165899e2ff7505712487b79 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 17 Dec 2020 16:32:13 +0000 Subject: [PATCH 049/289] All rubbish --- lib/html-proofer/element.rb | 2 +- lib/html-proofer/runner.rb | 6 +- .../2020/01/13/Citrix/index.html | 87 ------------------- spec/html-proofer/links_spec.rb | 7 -- 4 files changed, 5 insertions(+), 97 deletions(-) delete mode 100644 spec/html-proofer/fixtures/links/relative_nested_internal_hash/2020/01/13/Citrix/index.html diff --git a/lib/html-proofer/element.rb b/lib/html-proofer/element.rb index 9b09ef24..03ad8b65 100644 --- a/lib/html-proofer/element.rb +++ b/lib/html-proofer/element.rb @@ -158,7 +158,7 @@ def relative_link? end def link_points_to_same_page? - hash_link || param_link || internal_absolute_link? + hash_link || param_link end def hash_link diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index ec91e709..08480cf9 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -157,10 +157,12 @@ def validate_internal_urls @cache.write else @internal_urls.values.flatten.each do |internal_url| - @internal_link_checks.check_internal_link(internal_url.link, internal_url.path, internal_url.line, internal_url.content) + result = @internal_link_checks.check_internal_link(internal_url.link, internal_url.path, internal_url.line, internal_url.content) + next if result + + @failures.concat(@internal_link_checks.issues) unless @internal_link_checks.issues.length.zero? end end - @failures.concat(@internal_link_checks.issues) unless @internal_urls.length.zero? end def files diff --git a/spec/html-proofer/fixtures/links/relative_nested_internal_hash/2020/01/13/Citrix/index.html b/spec/html-proofer/fixtures/links/relative_nested_internal_hash/2020/01/13/Citrix/index.html deleted file mode 100644 index 291c145f..00000000 --- a/spec/html-proofer/fixtures/links/relative_nested_internal_hash/2020/01/13/Citrix/index.html +++ /dev/null @@ -1,87 +0,0 @@ - - - - - - - Wijd verspreide kwetsbaarheid in Citrix Gateway en Citrix Application Delivery Controller | Nederlands Security Meldpunt - - - - - - - - - - - - - - - - - - Skip to the content. - - - -
-

Nederlands Security Meldpunt -

-

Het Nederlandse meldpunt voor kwetsbaarheden, infecties en andere (cyber)security gerelateerde zaken. -

-
- -
-
-

Wijd verspreide kwetsbaarheid in Citrix Gateway en Citrix Application Delivery Controller -

-

- 13 Jan 2020 - -
- English below -

-

Het probleem -

-

Houdt er rekening mee dat apparaten al voor het uitvoeren van de patch door een aanvaller overgenomen kunnen zijn. -

-

Update: bevat meer informatie over hoe je kunt controleren of je Citrix ADC is overgenomen. -

-

Wat wij doen… -

-

Vanochtend ca 11:00 waren er bij het Security Meldpunt 546 via het publieke internet bereikbare kwetsbare Citrix ADC servers bekend. Wij zijn op dit moment actief bezig de beheerders van de netwerken waar deze servers in staan op de hoogte te stellen. -

-
-

- English -

-

The problem -

-

Please take note that it is possible that devices that did not have patches or mitigations applied may already have been taken over by an attacker. -

-

Update: contains information on how to check if/how your Citrix AC is compromised. -

-

What we are doing. -

-

As of this morning around 11:00 we were aware of 546 publicly accessible vulnerable devices in The Netherlands. We are currently actively informing the owners of the networks containing vulnerable devices. -

-
-
-
-
-
-
-
-
-
- - diff --git a/spec/html-proofer/links_spec.rb b/spec/html-proofer/links_spec.rb index 4dda74b6..856e91ee 100644 --- a/spec/html-proofer/links_spec.rb +++ b/spec/html-proofer/links_spec.rb @@ -681,11 +681,4 @@ proofer = run_proofer(file, :file, allow_hash_href: true) expect(proofer.failed_tests).to eq [] end - - it 'works for hash references in self' do - dir = "#{FIXTURES_DIR}/links/relative_nested_internal_hash" - proofer = run_proofer(dir, :directory, allow_hash_href: true) - - expect(proofer.failed_tests).to eq [] - end end From 480c4716b5a013b11b1e785495c89876fb6a5143 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 17 Dec 2020 16:37:33 +0000 Subject: [PATCH 050/289] :gem: bump to 3.18.2 --- lib/html-proofer/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/version.rb b/lib/html-proofer/version.rb index 61daa4ed..cb200bd2 100644 --- a/lib/html-proofer/version.rb +++ b/lib/html-proofer/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HTMLProofer - VERSION = '3.18.1' + VERSION = '3.18.2' end From 575c6f5b6cfae9ca29639170c909eae42bfd5bb2 Mon Sep 17 00:00:00 2001 From: Sebastian Cohnen Date: Sun, 27 Dec 2020 12:25:46 +0100 Subject: [PATCH 051/289] allows Ruby 3.0 to be used --- .github/workflows/ci.yml | 2 +- .github/workflows/lint.yml | 2 +- html-proofer.gemspec | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ba668e5..d519718d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: [2.7.2, 2.6.6, 2.5.8, 2.4.10] + ruby-version: [3.0.0, 2.7.2, 2.6.6, 2.5.8, 2.4.10] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b5953b2e..fa9e7c11 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v2 - uses: ruby/setup-ruby@v1 with: - ruby-version: 2.7 + ruby-version: 3.0 bundler-cache: true - run: bundle install - name: Rubocop diff --git a/html-proofer.gemspec b/html-proofer.gemspec index b0e2c8b0..16454079 100644 --- a/html-proofer.gemspec +++ b/html-proofer.gemspec @@ -17,7 +17,7 @@ Gem::Specification.new do |gem| gem.executables = all_files.grep(%r{^bin/}) { |f| File.basename(f) } gem.test_files = gem.files.grep(%r{^(spec)/}) gem.require_paths = ['lib'] - gem.required_ruby_version = '~> 2.4' + gem.required_ruby_version = '>= 2.4.10' gem.add_dependency 'addressable', '~> 2.3' gem.add_dependency 'mercenary', '~> 0.3' From 68f4cae5ef67d589c297a8f5d565c89e050e60a5 Mon Sep 17 00:00:00 2001 From: Sebastian Cohnen Date: Mon, 28 Dec 2020 11:54:31 +0100 Subject: [PATCH 052/289] limits ruby version to be < 4.0 --- html-proofer.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html-proofer.gemspec b/html-proofer.gemspec index 16454079..ab4c5740 100644 --- a/html-proofer.gemspec +++ b/html-proofer.gemspec @@ -17,7 +17,7 @@ Gem::Specification.new do |gem| gem.executables = all_files.grep(%r{^bin/}) { |f| File.basename(f) } gem.test_files = gem.files.grep(%r{^(spec)/}) gem.require_paths = ['lib'] - gem.required_ruby_version = '>= 2.4.10' + gem.required_ruby_version = ['>= 2.4.10', '< 4.0'] gem.add_dependency 'addressable', '~> 2.3' gem.add_dependency 'mercenary', '~> 0.3' From 84362d6d228fdc19202a5fb8fe2c1aaf2cd601cb Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Tue, 29 Dec 2020 19:58:39 +0000 Subject: [PATCH 053/289] :gem: bump to 3.18.3 --- lib/html-proofer/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/version.rb b/lib/html-proofer/version.rb index cb200bd2..2d0fceb2 100644 --- a/lib/html-proofer/version.rb +++ b/lib/html-proofer/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HTMLProofer - VERSION = '3.18.2' + VERSION = '3.18.3' end From e1e02f622342d8b7921ea278d24d272a2c476eff Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 2 Jan 2021 19:08:24 +0000 Subject: [PATCH 054/289] Catch bad URLs --- lib/html-proofer/element.rb | 2 +- spec/html-proofer/fixtures/links/bad_formatting.html | 8 ++++++++ spec/html-proofer/links_spec.rb | 6 ++++++ 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 spec/html-proofer/fixtures/links/bad_formatting.html diff --git a/lib/html-proofer/element.rb b/lib/html-proofer/element.rb index 03ad8b65..79cad413 100644 --- a/lib/html-proofer/element.rb +++ b/lib/html-proofer/element.rb @@ -71,7 +71,7 @@ def url end def valid? - !parts.nil? + !parts.nil? && !parts.host.nil? && !parts.path.nil? end def parts diff --git a/spec/html-proofer/fixtures/links/bad_formatting.html b/spec/html-proofer/fixtures/links/bad_formatting.html new file mode 100644 index 00000000..6d09e924 --- /dev/null +++ b/spec/html-proofer/fixtures/links/bad_formatting.html @@ -0,0 +1,8 @@ + + + + + + OK + + diff --git a/spec/html-proofer/links_spec.rb b/spec/html-proofer/links_spec.rb index 856e91ee..b9c48e81 100644 --- a/spec/html-proofer/links_spec.rb +++ b/spec/html-proofer/links_spec.rb @@ -681,4 +681,10 @@ proofer = run_proofer(file, :file, allow_hash_href: true) expect(proofer.failed_tests).to eq [] end + + it 'does not crash on badly formatted urls' do + file = "#{FIXTURES_DIR}/links/bad_formatting.html" + proofer = run_proofer(file, :file) + expect(proofer.failed_tests.first).to match(/is an invalid URL/) + end end From 65057a5fb1190ac29de09a31cfb910b2c1391063 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 2 Jan 2021 19:10:58 +0000 Subject: [PATCH 055/289] :gem: bump to 3.18.4 --- lib/html-proofer/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/version.rb b/lib/html-proofer/version.rb index 2d0fceb2..5e01dd51 100644 --- a/lib/html-proofer/version.rb +++ b/lib/html-proofer/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HTMLProofer - VERSION = '3.18.3' + VERSION = '3.18.4' end From 72ca5102e3ae0e3b616237c9bc7f66d73818709e Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 2 Jan 2021 19:19:00 +0000 Subject: [PATCH 056/289] Rejigger test --- lib/html-proofer/check/links.rb | 5 +++++ lib/html-proofer/element.rb | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/html-proofer/check/links.rb b/lib/html-proofer/check/links.rb index 3c9322ec..5754b692 100644 --- a/lib/html-proofer/check/links.rb +++ b/lib/html-proofer/check/links.rb @@ -51,6 +51,11 @@ def run # curl/Typheous inaccurately return 404s for some links. cc https://git.io/vyCFx next if @link.respond_to?(:rel) && @link.rel == 'dns-prefetch' + unless @link.path? + add_issue("#{@link.href} is an invalid URL", line: line, content: content) + next + end + add_to_external_urls(@link.href || @link.src) next elsif @link.internal? diff --git a/lib/html-proofer/element.rb b/lib/html-proofer/element.rb index 79cad413..911412d7 100644 --- a/lib/html-proofer/element.rb +++ b/lib/html-proofer/element.rb @@ -71,7 +71,11 @@ def url end def valid? - !parts.nil? && !parts.host.nil? && !parts.path.nil? + !parts.nil? + end + + def path? + !parts.host.nil? && !parts.path.nil? end def parts From 3bb8d353e0420d79fc97cf06b40081495b871343 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 2 Jan 2021 19:21:28 +0000 Subject: [PATCH 057/289] :gem: bump to 3.18.5 --- lib/html-proofer/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/version.rb b/lib/html-proofer/version.rb index 5e01dd51..35c1038b 100644 --- a/lib/html-proofer/version.rb +++ b/lib/html-proofer/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HTMLProofer - VERSION = '3.18.4' + VERSION = '3.18.5' end From 777d834c19782ca6b377c4bfdefd57ce58138299 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sun, 21 Feb 2021 16:03:24 +0000 Subject: [PATCH 058/289] Correct link ignore regexp --- lib/html-proofer/element.rb | 2 ++ lib/html-proofer/middleware.rb | 2 +- spec/html-proofer/middleware_spec.rb | 40 ++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/lib/html-proofer/element.rb b/lib/html-proofer/element.rb index 911412d7..fc3a5f65 100644 --- a/lib/html-proofer/element.rb +++ b/lib/html-proofer/element.rb @@ -220,6 +220,8 @@ def absolute_path end def ignores_pattern_check(links) + return false unless links.is_a?(Array) + links.each do |ignore| case ignore when String diff --git a/lib/html-proofer/middleware.rb b/lib/html-proofer/middleware.rb index 3c2af938..1f4c995b 100644 --- a/lib/html-proofer/middleware.rb +++ b/lib/html-proofer/middleware.rb @@ -22,7 +22,7 @@ def self.options allow_hash_href: true, check_external_hash: true, check_html: true, - url_ignore: [/.*/], # Don't try to check if local files exist + url_ignore: [%r{^/}], # Don't try to check if local files exist validation: { report_eof_tags: true } } end diff --git a/spec/html-proofer/middleware_spec.rb b/spec/html-proofer/middleware_spec.rb index af608428..d622da81 100644 --- a/spec/html-proofer/middleware_spec.rb +++ b/spec/html-proofer/middleware_spec.rb @@ -32,6 +32,46 @@ end end + context 'options' do + let(:response_fixture) { File.join(File.absolute_path(FIXTURES_DIR), 'links', 'broken_root_link_internal.html') } + + before(:each) do + @default_opts = HTMLProofer::Middleware.options[:url_ignore] + end + + after(:each) do + HTMLProofer::Middleware.options[:url_ignore] = @default_opts + end + + let(:middleware_with_ignore) do + HTMLProofer::Middleware.options[:url_ignore] = [/broken/] + HTMLProofer::Middleware.new(app) + end + let(:middleware_with_no_ignore) do + HTMLProofer::Middleware.options[:url_ignore] = nil + HTMLProofer::Middleware.new(app) + end + + let(:subject) { middleware.call(request) } + let(:subject_with_ignore) { middleware_with_ignore.call(request) } + let(:subject_with_no_ignore) { middleware_with_no_ignore.call(request) } + context 'internal files' do + it 'does not raise an error with default options set' do + subject + end + + it 'does not raise an error with options explicitly set' do + subject_with_ignore + end + + it 'does raise an error with options removed' do + expect do + subject_with_no_ignore + end.to raise_error(HTMLProofer::Middleware::InvalidHtmlError) + end + end + end + context 'proofer-ignore' do let(:skip_request) { { 'REQUEST_METHOD' => 'GET', 'QUERY_STRING' => 'proofer-ignore' } } let(:subject) { middleware.call(skip_request) } From aa60148c53095e28e4ca0f4e9d43a4fefe80fc27 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sun, 21 Feb 2021 16:33:02 +0000 Subject: [PATCH 059/289] Handle internal & external cache merging --- lib/html-proofer/cache.rb | 7 +- lib/html-proofer/check/links.rb | 7 +- lib/html-proofer/element.rb | 2 +- lib/html-proofer/runner.rb | 4 +- lib/html-proofer/url_validator.rb | 2 +- spec/html-proofer/cache_spec.rb | 54 +++++++++++++ .../html-proofer/fixtures/cache/.external.log | 1 + .../fixtures/cache/.internal_and_external.log | 1 + .../fixtures/cache/external_example.html | 11 +++ .../cache/internal_and_external_example.html | 13 ++++ .../cache_log_level_error_type_file_.yml | 76 +++++++++++++++++++ .../cache_log_level_error_type_file_.yml | 76 +++++++++++++++++++ 12 files changed, 244 insertions(+), 10 deletions(-) create mode 100644 spec/html-proofer/fixtures/cache/.external.log create mode 100644 spec/html-proofer/fixtures/cache/.internal_and_external.log create mode 100644 spec/html-proofer/fixtures/cache/external_example.html create mode 100644 spec/html-proofer/fixtures/cache/internal_and_external_example.html create mode 100644 spec/html-proofer/fixtures/vcr_cassettes/cache/external_example_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_external_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml create mode 100644 spec/html-proofer/fixtures/vcr_cassettes/cache/internal_and_external_example_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_internal_and_external_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml diff --git a/lib/html-proofer/cache.rb b/lib/html-proofer/cache.rb index 2e143f79..18304320 100644 --- a/lib/html-proofer/cache.rb +++ b/lib/html-proofer/cache.rb @@ -109,8 +109,13 @@ def detect_url_changes(found) additions end + # TODO: Garbage performance--both the external and internal + # caches need access to this file. Write a proper versioned + # schema in the future def write - File.write(cache_file, @cache_log.to_json) + file = {} + file = JSON.parse(File.read(cache_file)) if File.exist?(cache_file) + File.write(cache_file, file.merge(@cache_log).to_json) end def load? diff --git a/lib/html-proofer/check/links.rb b/lib/html-proofer/check/links.rb index 5754b692..27b7b3a2 100644 --- a/lib/html-proofer/check/links.rb +++ b/lib/html-proofer/check/links.rb @@ -59,11 +59,8 @@ def run add_to_external_urls(@link.href || @link.src) next elsif @link.internal? - if @link.exists? || @link.hash - add_to_internal_urls(@link.href, InternalLink.new(@link, @path, line, content)) - else - add_issue("internally linking to #{@link.href}, which does not exist", line: line, content: content) - end + add_to_internal_urls(@link.href, InternalLink.new(@link, @path, line, content)) + add_issue("internally linking to #{@link.href}, which does not exist", line: line, content: content) if !@link.exists? && !@link.hash end end diff --git a/lib/html-proofer/element.rb b/lib/html-proofer/element.rb index fc3a5f65..ab781c51 100644 --- a/lib/html-proofer/element.rb +++ b/lib/html-proofer/element.rb @@ -24,7 +24,7 @@ def initialize(obj, check, logger) raise e end - @aria_hidden = defined?(@aria_hidden) && @aria_hidden == 'true' ? true : false + @aria_hidden = defined?(@aria_hidden) && @aria_hidden == 'true' @data_proofer_ignore = defined?(@data_proofer_ignore) diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index 08480cf9..dde9e157 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -63,7 +63,7 @@ def check_list_of_links swap(url, @options[:url_swap]) end end - @external_urls = Hash[*@src.map { |s| [s, nil] }.flatten] + @external_urls = @src.map { |s| [s, nil] }.flatten.to_h validate_external_urls end @@ -123,7 +123,7 @@ def check_parsed(html, path) end external_urls = check.external_urls - external_urls = Hash[check.external_urls.map { |url, file| [swap(url, @options[:url_swap]), file] }] if @options[:url_swap] + external_urls = check.external_urls.map { |url, file| [swap(url, @options[:url_swap]), file] }.to_h if @options[:url_swap] result[:external_urls].merge!(external_urls) result[:failures].concat(check.issues) end diff --git a/lib/html-proofer/url_validator.rb b/lib/html-proofer/url_validator.rb index 4cf85c11..4f1e3b16 100644 --- a/lib/html-proofer/url_validator.rb +++ b/lib/html-proofer/url_validator.rb @@ -85,7 +85,7 @@ def extract_domain_path(uri) # for HEAD. If we've decided to check for hashes, we must do a GET--HEAD is # not available as an option. def external_link_checker(external_urls) - external_urls = Hash[external_urls.sort] + external_urls = external_urls.sort.to_h count = external_urls.length check_text = pluralize(count, 'external link', 'external links') diff --git a/spec/html-proofer/cache_spec.rb b/spec/html-proofer/cache_spec.rb index 7c5b97a0..1ec5489a 100644 --- a/spec/html-proofer/cache_spec.rb +++ b/spec/html-proofer/cache_spec.rb @@ -247,4 +247,58 @@ def read_cache(cache_file) Timecop.return end end + + it 'does recheck failures, regardless of external-only cache' do + cache_file_name = '.external.log' + cache_location = File.join(storage_dir, cache_file_name) + + file = "#{FIXTURES_DIR}/cache/external_example.html" + + new_time = Time.local(2021, 0o1, 27, 12, 0, 0) + Timecop.freeze(new_time) + + File.delete(cache_location) if File.exist?(cache_location) + + run_proofer(file, :file, external_only: true, links_only: true, cache: { timeframe: '1d', cache_file: cache_file_name }.merge(default_cache_options)) + + cache = JSON.parse(File.read(cache_location)) + + expect(cache.keys.count).to eq(1) + expect(cache.keys[0]).to eq('https://github.com/gjtorikian/html-proofer') + + run_proofer(file, :file, links_only: true, cache: { timeframe: '1d', cache_file: cache_file_name }.merge(default_cache_options)) + + cache = JSON.parse(File.read(cache_location)) + expect(cache.keys.count).to eq(1) + expect(cache.keys[0]).to eq('https://github.com/gjtorikian/html-proofer') + + Timecop.return + end + + it 'does recheck failures, regardless of external and internal cache' do + cache_file_name = '.internal_and_external.log' + cache_location = File.join(storage_dir, cache_file_name) + + file = "#{FIXTURES_DIR}/cache/internal_and_external_example.html" + + new_time = Time.local(2021, 0o1, 27, 12, 0, 0) + Timecop.freeze(new_time) + + File.delete(cache_location) if File.exist?(cache_location) + + run_proofer(file, :file, external_only: true, links_only: true, cache: { timeframe: '1d', cache_file: cache_file_name }.merge(default_cache_options)) + + cache = JSON.parse(File.read(cache_location)) + + expect(cache.keys.count).to eq(1) + expect(cache.keys[0]).to eq('https://github.com/gjtorikian/html-proofer') + + run_proofer(file, :file, links_only: true, cache: { timeframe: '1d', cache_file: cache_file_name }.merge(default_cache_options)) + + cache = JSON.parse(File.read(cache_location)) + expect(cache.keys.count).to eq(2) + expect(cache.keys[0]).to eq('https://github.com/gjtorikian/html-proofer') + + Timecop.return + end end diff --git a/spec/html-proofer/fixtures/cache/.external.log b/spec/html-proofer/fixtures/cache/.external.log new file mode 100644 index 00000000..d147adeb --- /dev/null +++ b/spec/html-proofer/fixtures/cache/.external.log @@ -0,0 +1 @@ +{"https://github.com/gjtorikian/html-proofer":{"time":"2021-01-27 12:00:00 +0000","filenames":["spec/html-proofer/fixtures/cache/external_example.html"],"status":200,"message":""}} \ No newline at end of file diff --git a/spec/html-proofer/fixtures/cache/.internal_and_external.log b/spec/html-proofer/fixtures/cache/.internal_and_external.log new file mode 100644 index 00000000..17c57ce2 --- /dev/null +++ b/spec/html-proofer/fixtures/cache/.internal_and_external.log @@ -0,0 +1 @@ +{"https://github.com/gjtorikian/html-proofer":{"time":"2021-01-27 12:00:00 +0000","filenames":["spec/html-proofer/fixtures/cache/internal_and_external_example.html"],"status":200,"message":""},"/somewhere.html":{"time":"2021-01-27 12:00:00 +0000","filenames":["spec/html-proofer/fixtures/cache/internal_and_external_example.html"],"status":200,"message":""}} \ No newline at end of file diff --git a/spec/html-proofer/fixtures/cache/external_example.html b/spec/html-proofer/fixtures/cache/external_example.html new file mode 100644 index 00000000..09c7add9 --- /dev/null +++ b/spec/html-proofer/fixtures/cache/external_example.html @@ -0,0 +1,11 @@ + + + + Example + + + The amazing HTMLProofer + + diff --git a/spec/html-proofer/fixtures/cache/internal_and_external_example.html b/spec/html-proofer/fixtures/cache/internal_and_external_example.html new file mode 100644 index 00000000..1e486d38 --- /dev/null +++ b/spec/html-proofer/fixtures/cache/internal_and_external_example.html @@ -0,0 +1,13 @@ + + + + Example + + + The amazing HTMLProofer + + Somewhere + + diff --git a/spec/html-proofer/fixtures/vcr_cassettes/cache/external_example_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_external_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml b/spec/html-proofer/fixtures/vcr_cassettes/cache/external_example_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_external_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml new file mode 100644 index 00000000..07a15211 --- /dev/null +++ b/spec/html-proofer/fixtures/vcr_cassettes/cache/external_example_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_external_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml @@ -0,0 +1,76 @@ +--- +http_interactions: +- request: + method: head + uri: https://github.com/gjtorikian/html-proofer + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.5; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + server: + - GitHub.com + date: + - Sun, 21 Feb 2021 16:32:07 GMT + content-type: + - text/html; charset=utf-8 + vary: + - X-PJAX, Accept-Encoding, Accept, X-Requested-With + etag: + - W/"caacf53a5a6580b9579b48c4e7c35847" + cache-control: + - max-age=0, private, must-revalidate + strict-transport-security: + - max-age=31536000; includeSubdomains; preload + x-frame-options: + - deny + x-content-type-options: + - nosniff + x-xss-protection: + - 1; mode=block + referrer-policy: + - no-referrer-when-downgrade + expect-ct: + - max-age=2592000, report-uri="https://api.github.com/_private/browser/errors" + content-security-policy: + - 'default-src ''none''; base-uri ''self''; block-all-mixed-content; connect-src + ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com + github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com + github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com + cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com online.visualstudio.com/api/v1/locations; + font-src github.githubassets.com; form-action ''self'' github.com gist.github.com; + frame-ancestors ''none''; frame-src render.githubusercontent.com; img-src + ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com + github-cloud.s3.amazonaws.com user-images.githubusercontent.com/ *.githubusercontent.com; + manifest-src ''self''; media-src ''none''; script-src github.githubassets.com; + style-src ''unsafe-inline'' github.githubassets.com; worker-src github.com/socket-worker-5029ae85.js + gist.github.com/socket-worker-5029ae85.js' + set-cookie: + - _gh_sess=FVxpDGIyyH6A21E%2FHQIg7c18Or8RAQkAqP7S77pa12Gdkt2ht0muVqUEqOFoBCJJI8izF1QaH7NCYDWku3Hk50ic5uaGJnjAPFWI2kX5q6OYQhseU33bO7q0r%2Fl6kdojyfURMLs8Ic7zzZ%2BJxdLlZh8xS0KEeVpX1omvlULMT169y0322Wrsw9YtBBgEtMHinHsyWBDkjSwZxh5y9%2BW7jFx4vPMvBlnao7RTHRAoSPQ3SFW2SdrE76IuSFt89Y7VGOn0EKD%2Fb71JU0Byv8xK1w%3D%3D--8NQJ2ZGl9U%2FqC3M5--1dw7YpQmajE2qqT5%2FQDmug%3D%3D; + Path=/; HttpOnly; Secure; SameSite=Lax + - _octo=GH1.1.168931837.1613925128; Path=/; Domain=github.com; Expires=Mon, + 21 Feb 2022 16:32:08 GMT; Secure; SameSite=Lax + - logged_in=no; Path=/; Domain=github.com; Expires=Mon, 21 Feb 2022 16:32:08 + GMT; HttpOnly; Secure; SameSite=Lax + accept-ranges: + - bytes + x-github-request-id: + - EA4B:61B0:217BDFF:269EF9E:60328B08 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://github.com/gjtorikian/html-proofer + recorded_at: Wed, 27 Jan 2021 12:00:00 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html-proofer/fixtures/vcr_cassettes/cache/internal_and_external_example_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_internal_and_external_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml b/spec/html-proofer/fixtures/vcr_cassettes/cache/internal_and_external_example_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_internal_and_external_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml new file mode 100644 index 00000000..7af71169 --- /dev/null +++ b/spec/html-proofer/fixtures/vcr_cassettes/cache/internal_and_external_example_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_internal_and_external_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml @@ -0,0 +1,76 @@ +--- +http_interactions: +- request: + method: head + uri: https://github.com/gjtorikian/html-proofer + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.5; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + server: + - GitHub.com + date: + - Sun, 21 Feb 2021 16:32:07 GMT + content-type: + - text/html; charset=utf-8 + vary: + - X-PJAX, Accept-Encoding, Accept, X-Requested-With + etag: + - W/"caacf53a5a6580b9579b48c4e7c35847" + cache-control: + - max-age=0, private, must-revalidate + strict-transport-security: + - max-age=31536000; includeSubdomains; preload + x-frame-options: + - deny + x-content-type-options: + - nosniff + x-xss-protection: + - 1; mode=block + referrer-policy: + - no-referrer-when-downgrade + expect-ct: + - max-age=2592000, report-uri="https://api.github.com/_private/browser/errors" + content-security-policy: + - 'default-src ''none''; base-uri ''self''; block-all-mixed-content; connect-src + ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com + github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com + github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com + cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com online.visualstudio.com/api/v1/locations; + font-src github.githubassets.com; form-action ''self'' github.com gist.github.com; + frame-ancestors ''none''; frame-src render.githubusercontent.com; img-src + ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com + github-cloud.s3.amazonaws.com user-images.githubusercontent.com/ *.githubusercontent.com; + manifest-src ''self''; media-src ''none''; script-src github.githubassets.com; + style-src ''unsafe-inline'' github.githubassets.com; worker-src github.com/socket-worker-5029ae85.js + gist.github.com/socket-worker-5029ae85.js' + set-cookie: + - _gh_sess=46RoTpx1xZeM6V2gyQ6RUJMiXmvy%2BbxrEcxPIktSfACs4M3XPgtmOKjITU9cFKOkqb21u%2FA25dQxwAv7y7KInm6XJK3P6xEc9uPTU65y8vpt%2FRN9QK9cfcEyhRQ2k7T1TRRltkalm0G3nOhNZnb1EoUcibLEuDYkJwpTFVBagH55cOSsINz%2BjwLd4dd4BO6k3tV2HjKE6YVHZdGeaq1sIxth7n8LDrp8916Ygm0TAVy7UG71UZC2cQPwz0pRu9VwYUpuDc0PzXWNwsz2Ey74bg%3D%3D--jSHIFDa3UTkzqp%2BP--adTIajHGBxFj%2FQ332TDFrw%3D%3D; + Path=/; HttpOnly; Secure; SameSite=Lax + - _octo=GH1.1.1571982095.1613925127; Path=/; Domain=github.com; Expires=Mon, + 21 Feb 2022 16:32:07 GMT; Secure; SameSite=Lax + - logged_in=no; Path=/; Domain=github.com; Expires=Mon, 21 Feb 2022 16:32:07 + GMT; HttpOnly; Secure; SameSite=Lax + accept-ranges: + - bytes + x-github-request-id: + - EA49:128D6:119821F:145FC7E:60328B07 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://github.com/gjtorikian/html-proofer + recorded_at: Wed, 27 Jan 2021 12:00:00 GMT +recorded_with: VCR 2.9.3 From 8a186cb09d1c31763d402432ceacf69b976ba5e6 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sun, 21 Feb 2021 16:35:31 +0000 Subject: [PATCH 060/289] typo --- lib/html-proofer/runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index dde9e157..91c6ecf6 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -63,7 +63,7 @@ def check_list_of_links swap(url, @options[:url_swap]) end end - @external_urls = @src.map { |s| [s, nil] }.flatten.to_h + @external_urls = [@src.map { |s| [s, nil] }.flatten].to_h validate_external_urls end From 580c6c11066c3def286dffd71dadf817cb29c176 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sun, 21 Feb 2021 16:41:37 +0000 Subject: [PATCH 061/289] each_with_object forever --- lib/html-proofer/runner.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index 91c6ecf6..9b858968 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -63,7 +63,9 @@ def check_list_of_links swap(url, @options[:url_swap]) end end - @external_urls = [@src.map { |s| [s, nil] }.flatten].to_h + @external_urls = @src.each_with_object({}) do |url, hash| + hash[url]= nil + end validate_external_urls end From ed39330e9aabd5608884b9d430e1d14a72125e3b Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sun, 21 Feb 2021 16:43:01 +0000 Subject: [PATCH 062/289] lint --- lib/html-proofer/runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index 9b858968..4ce75018 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -64,7 +64,7 @@ def check_list_of_links end end @external_urls = @src.each_with_object({}) do |url, hash| - hash[url]= nil + hash[url] = nil end validate_external_urls end From 75ad787210767ab83aa5c6597662088369b9f105 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sun, 21 Feb 2021 16:43:42 +0000 Subject: [PATCH 063/289] :gem: bump to 3.18.6 --- lib/html-proofer/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/version.rb b/lib/html-proofer/version.rb index 35c1038b..f0a75c53 100644 --- a/lib/html-proofer/version.rb +++ b/lib/html-proofer/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HTMLProofer - VERSION = '3.18.5' + VERSION = '3.18.6' end From c9128c4706e18054ab6d3e86a09750eb7435e1bf Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Sat, 27 Feb 2021 00:06:37 +0000 Subject: [PATCH 064/289] Do a better job preserving cache logic --- lib/html-proofer/cache.rb | 14 ++-- lib/html-proofer/runner.rb | 2 +- lib/html-proofer/url_validator.rb | 2 +- spec/html-proofer/cache_spec.rb | 29 ++++++- .../fixtures/cache/.removed_link.log | 1 + .../fixtures/cache/some_link.html | 9 +++ .../cache_log_level_error_type_file_.yml | 76 +++++++++++++++++++ .../cache_log_level_error_type_file_.yml | 76 +++++++++++++++++++ 8 files changed, 198 insertions(+), 11 deletions(-) create mode 100644 spec/html-proofer/fixtures/cache/.removed_link.log create mode 100644 spec/html-proofer/fixtures/cache/some_link.html create mode 100644 spec/html-proofer/fixtures/vcr_cassettes/cache/external_example_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_removed_link_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml create mode 100644 spec/html-proofer/fixtures/vcr_cassettes/cache/some_link_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_removed_link_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml diff --git a/lib/html-proofer/cache.rb b/lib/html-proofer/cache.rb index 18304320..9048588b 100644 --- a/lib/html-proofer/cache.rb +++ b/lib/html-proofer/cache.rb @@ -71,7 +71,7 @@ def add(url, filenames, status, msg = '') @cache_log[clean_url(url)] = data end - def detect_url_changes(found) + def detect_url_changes(found, type) existing_urls = @cache_log.keys.map { |url| clean_url(url) } found_urls = found.keys.map { |url| clean_url(url) } @@ -92,11 +92,13 @@ def detect_url_changes(found) # remove from cache URLs that no longer exist del = 0 + uri_regexp = URI::DEFAULT_PARSER.make_regexp @cache_log.delete_if do |url, _| url = clean_url(url) + if found_urls.include?(url) false - else + elsif (type == :internal && url !~ uri_regexp) || (type == :external && url =~ uri_regexp) @logger.log :debug, "Removing #{url} from cache check" del += 1 true @@ -113,17 +115,15 @@ def detect_url_changes(found) # caches need access to this file. Write a proper versioned # schema in the future def write - file = {} - file = JSON.parse(File.read(cache_file)) if File.exist?(cache_file) - File.write(cache_file, file.merge(@cache_log).to_json) + File.write(cache_file, @cache_log.to_json) end def load? @load.nil? end - def retrieve_urls(urls) - urls_to_check = detect_url_changes(urls) + def retrieve_urls(urls, type) + urls_to_check = detect_url_changes(urls, type) @cache_log.each_pair do |url, cache| next if within_timeframe?(cache['time']) && cache['message'].empty? # these were successes to skip diff --git a/lib/html-proofer/runner.rb b/lib/html-proofer/runner.rb index 4ce75018..c30e5d23 100644 --- a/lib/html-proofer/runner.rb +++ b/lib/html-proofer/runner.rb @@ -238,7 +238,7 @@ def before_request(&block) end def load_internal_cache - urls_to_check = @cache.retrieve_urls(@internal_urls) + urls_to_check = @cache.retrieve_urls(@internal_urls, :internal) cache_text = pluralize(urls_to_check.count, 'internal link', 'internal links') @logger.log :info, "Found #{cache_text} in the cache..." diff --git a/lib/html-proofer/url_validator.rb b/lib/html-proofer/url_validator.rb index 4f1e3b16..ee06546d 100644 --- a/lib/html-proofer/url_validator.rb +++ b/lib/html-proofer/url_validator.rb @@ -26,7 +26,7 @@ def run @external_urls = remove_query_values if @cache.use_cache? - urls_to_check = @cache.retrieve_urls(@external_urls) + urls_to_check = @cache.retrieve_urls(@external_urls, :external) external_link_checker(urls_to_check) @cache.write else diff --git a/spec/html-proofer/cache_spec.rb b/spec/html-proofer/cache_spec.rb index 1ec5489a..9b04b649 100644 --- a/spec/html-proofer/cache_spec.rb +++ b/spec/html-proofer/cache_spec.rb @@ -89,9 +89,9 @@ def read_cache(cache_file) let(:cache_file_name) { '.htmlproofer.log' } it 'knows how to write to cache' do - broken_link_external_filepath = "#{FIXTURES_DIR}/links/broken_link_external.html" + missing_link_href = "#{FIXTURES_DIR}/links/missing_link_href.html" expect_any_instance_of(HTMLProofer::Cache).to receive(:write).twice # once for internal, once for external - run_proofer(broken_link_external_filepath, :file, cache: { timeframe: '30d', cache_file: cache_file_name }.merge(default_cache_options)) + run_proofer(missing_link_href, :file, cache: { timeframe: '30d', cache_file: cache_file_name }.merge(default_cache_options)) log = read_cache(cache_file) expect(log.keys.length).to eq(2) @@ -301,4 +301,29 @@ def read_cache(cache_file) Timecop.return end + + it 'removes links that do not exist' do + cache_file_name = '.removed_link.log' + cache_location = File.join(storage_dir, cache_file_name) + + new_time = Time.local(2021, 0o1, 27, 12, 0, 0) + Timecop.freeze(new_time) + + File.delete(cache_location) if File.exist?(cache_location) + + run_proofer("#{FIXTURES_DIR}/cache/external_example.html", :file, external_only: true, links_only: true, cache: { timeframe: '1d', cache_file: cache_file_name }.merge(default_cache_options)) + + cache = JSON.parse(File.read(cache_location)) + + expect(cache.keys.count).to eq(1) + expect(cache.keys[0]).to eq('https://github.com/gjtorikian/html-proofer') + + run_proofer("#{FIXTURES_DIR}/cache/some_link.html", :file, external_only: true, links_only: true, cache: { timeframe: '1d', cache_file: cache_file_name }.merge(default_cache_options)) + + cache = JSON.parse(File.read(cache_location)) + expect(cache.keys.count).to eq(1) + expect(cache.keys[0]).to eq('https://github.com/gjtorikian') + + Timecop.return + end end diff --git a/spec/html-proofer/fixtures/cache/.removed_link.log b/spec/html-proofer/fixtures/cache/.removed_link.log new file mode 100644 index 00000000..7de644c8 --- /dev/null +++ b/spec/html-proofer/fixtures/cache/.removed_link.log @@ -0,0 +1 @@ +{"https://github.com/gjtorikian":{"time":"2021-01-27 12:00:00 +0000","filenames":["spec/html-proofer/fixtures/cache/some_link.html"],"status":200,"message":""}} \ No newline at end of file diff --git a/spec/html-proofer/fixtures/cache/some_link.html b/spec/html-proofer/fixtures/cache/some_link.html new file mode 100644 index 00000000..209b46c8 --- /dev/null +++ b/spec/html-proofer/fixtures/cache/some_link.html @@ -0,0 +1,9 @@ + + + + Example + + + The amazing gjtorikian + + diff --git a/spec/html-proofer/fixtures/vcr_cassettes/cache/external_example_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_removed_link_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml b/spec/html-proofer/fixtures/vcr_cassettes/cache/external_example_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_removed_link_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml new file mode 100644 index 00000000..0e195d5a --- /dev/null +++ b/spec/html-proofer/fixtures/vcr_cassettes/cache/external_example_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_removed_link_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml @@ -0,0 +1,76 @@ +--- +http_interactions: +- request: + method: head + uri: https://github.com/gjtorikian/html-proofer + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.6; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + server: + - GitHub.com + date: + - Sat, 27 Feb 2021 00:06:16 GMT + content-type: + - text/html; charset=utf-8 + vary: + - X-PJAX, Accept-Encoding, Accept, X-Requested-With + etag: + - W/"86ae8e298976293559b60f77a73bd6b9" + cache-control: + - max-age=0, private, must-revalidate + strict-transport-security: + - max-age=31536000; includeSubdomains; preload + x-frame-options: + - deny + x-content-type-options: + - nosniff + x-xss-protection: + - 1; mode=block + referrer-policy: + - no-referrer-when-downgrade + expect-ct: + - max-age=2592000, report-uri="https://api.github.com/_private/browser/errors" + content-security-policy: + - 'default-src ''none''; base-uri ''self''; block-all-mixed-content; connect-src + ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com + github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com + github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com + cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com online.visualstudio.com/api/v1/locations; + font-src github.githubassets.com; form-action ''self'' github.com gist.github.com; + frame-ancestors ''none''; frame-src render.githubusercontent.com; img-src + ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com + github-cloud.s3.amazonaws.com user-images.githubusercontent.com/ *.githubusercontent.com; + manifest-src ''self''; media-src ''none''; script-src github.githubassets.com; + style-src ''unsafe-inline'' github.githubassets.com; worker-src github.com/socket-worker-5029ae85.js + gist.github.com/socket-worker-5029ae85.js' + set-cookie: + - _gh_sess=0mwdcK%2BqmeQrngjFOA5Sr4wCgGP5UOAM9Ri%2FW9xJLa2bnKEtE8kw%2F36NlEbmVSsQ604Ud6lpuAKhUrTkXYwVR527QnCWPK%2B4YVkEt8%2BqMglb4zmObChj9%2FkiiXsktkV6zLiMb5H7PcxxUNoOOH7ob7J6XV25eL4sn9R3kC9gbQQIam9UyrFWzEHUvkxQm1%2F3O%2B9DIaLvsXfm4j4wWQKkRHWlM0HuRkDeHL79HaErJD3182DwlGuEp3sKXghNUUQOTO5DOOmNdWaajwqGTSrG%2FA%3D%3D--rO1FBF9Tkdisq3i7--goYWfNuKdFCzNeqPj8VzUA%3D%3D; + Path=/; HttpOnly; Secure; SameSite=Lax + - _octo=GH1.1.1543019191.1614384376; Path=/; Domain=github.com; Expires=Sun, + 27 Feb 2022 00:06:16 GMT; Secure; SameSite=Lax + - logged_in=no; Path=/; Domain=github.com; Expires=Sun, 27 Feb 2022 00:06:16 + GMT; HttpOnly; Secure; SameSite=Lax + accept-ranges: + - bytes + x-github-request-id: + - E50D:201A:229CA2:323A9D:60398CF7 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://github.com/gjtorikian/html-proofer + recorded_at: Wed, 27 Jan 2021 12:00:00 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html-proofer/fixtures/vcr_cassettes/cache/some_link_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_removed_link_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml b/spec/html-proofer/fixtures/vcr_cassettes/cache/some_link_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_removed_link_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml new file mode 100644 index 00000000..8ecc7e59 --- /dev/null +++ b/spec/html-proofer/fixtures/vcr_cassettes/cache/some_link_html_external_only_true_links_only_true_cache_timeframe_1d_cache_file_removed_link_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml @@ -0,0 +1,76 @@ +--- +http_interactions: +- request: + method: head + uri: https://github.com/gjtorikian + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.6; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + server: + - GitHub.com + date: + - Sat, 27 Feb 2021 00:06:17 GMT + content-type: + - text/html; charset=utf-8 + vary: + - X-Requested-With, Accept-Encoding, Accept, X-Requested-With + etag: + - W/"f4ee130590c21215b6f14377520e1555" + cache-control: + - max-age=0, private, must-revalidate + strict-transport-security: + - max-age=31536000; includeSubdomains; preload + x-frame-options: + - deny + x-content-type-options: + - nosniff + x-xss-protection: + - 1; mode=block + referrer-policy: + - origin-when-cross-origin, strict-origin-when-cross-origin + expect-ct: + - max-age=2592000, report-uri="https://api.github.com/_private/browser/errors" + content-security-policy: + - 'default-src ''none''; base-uri ''self''; block-all-mixed-content; connect-src + ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com + github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com + github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com + cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com; font-src + github.githubassets.com; form-action ''self'' github.com gist.github.com; + frame-ancestors ''none''; frame-src render.githubusercontent.com; img-src + ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com + github-cloud.s3.amazonaws.com user-images.githubusercontent.com/ *.githubusercontent.com; + manifest-src ''self''; media-src ''none''; script-src github.githubassets.com; + style-src ''unsafe-inline'' github.githubassets.com; worker-src github.com/socket-worker-5029ae85.js + gist.github.com/socket-worker-5029ae85.js' + set-cookie: + - _gh_sess=FIJCF4%2BirdhL1zk0v56AUrhulYRb7ori%2BjHgbibG%2Bx4W%2BAFLy478DlJTVR7dCe5AYul0doZAYWottUzqRbCux3edTV0KVBeJ04e%2B5JPtUt6S22n3F9xFTMwuE4WMtrCI49NKe%2FFCOVRBErmjjFItlm0fVpbHFWi4Vti0efT9B%2FYh2vu1jX934JE6NdfwNYlcfSmh5HekU%2B3EfbCDmJ0A00KnSzG5767KvxRc%2By1k30rCGOPTFoSPZ1Bu8ygWbx6zLXR515D%2F405TYomAiYlg4w%3D%3D--BwGiprDR7COosTV8--BCgboS8KvU4p4do82CCq8A%3D%3D; + Path=/; HttpOnly; Secure; SameSite=Lax + - _octo=GH1.1.1147749894.1614384376; Path=/; Domain=github.com; Expires=Sun, + 27 Feb 2022 00:06:16 GMT; Secure; SameSite=Lax + - logged_in=no; Path=/; Domain=github.com; Expires=Sun, 27 Feb 2022 00:06:16 + GMT; HttpOnly; Secure; SameSite=Lax + accept-ranges: + - bytes + x-github-request-id: + - E510:140A:661090:7C85F4:60398CF8 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://github.com/gjtorikian + recorded_at: Wed, 27 Jan 2021 12:00:00 GMT +recorded_with: VCR 2.9.3 From 45c85f4d5b0bfc0da7b02682e081063118e3c3b0 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 4 Mar 2021 07:54:43 -0500 Subject: [PATCH 065/289] Demonstrate runner/cache conflict --- spec/html-proofer/cache_spec.rb | 4 ++-- spec/html-proofer/runner_spec.rb | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/spec/html-proofer/cache_spec.rb b/spec/html-proofer/cache_spec.rb index 9b04b649..a23d0378 100644 --- a/spec/html-proofer/cache_spec.rb +++ b/spec/html-proofer/cache_spec.rb @@ -136,9 +136,9 @@ def read_cache(cache_file) expect_any_instance_of(HTMLProofer::Cache).to receive(:write) # we expect an add since we are mocking outside the timeframe - expect_any_instance_of(HTMLProofer::Cache).to receive(:add).with('www.github.com', nil, 200) + expect_any_instance_of(HTMLProofer::Cache).to receive(:add).with('https://www.github.com', nil, 200) - run_proofer(['www.github.com'], :links, cache: { timeframe: '4d', cache_file: cache_file_name }.merge(default_cache_options)) + run_proofer(['https://www.github.com'], :links, cache: { timeframe: '4d', cache_file: cache_file_name }.merge(default_cache_options)) Timecop.return end diff --git a/spec/html-proofer/runner_spec.rb b/spec/html-proofer/runner_spec.rb index cf4ee985..e6648e6b 100644 --- a/spec/html-proofer/runner_spec.rb +++ b/spec/html-proofer/runner_spec.rb @@ -25,5 +25,35 @@ expect(request.options).to include(:headers) expect(request.options[:headers]).to include('Authorization' => auth) end + + it 'plays nice with cache' do + cache_file_name = '.runner.log' + storage_dir = File.join(FIXTURES_DIR, '/cache') + cache_location = File.join(storage_dir, cache_file_name) + + File.delete(cache_location) if File.exist?(cache_location) + + opts = { + cache: { timeframe: '1d', cache_file: cache_file_name, storage_dir: storage_dir } + } + dir = "#{FIXTURES_DIR}/links/_site" + proofer = make_proofer(dir, :directory, opts) + request = nil + auth = 'Bearer ' + proofer.before_request do |r| + r.options[:headers]['Authorization'] = auth + request = r + end + + cassette_name = make_cassette_name(dir, opts) + VCR.use_cassette(cassette_name, record: :new_episodes) do + capture_stderr { proofer.run } + proofer + end + + expect(request).to respond_to(:options) + expect(request.options).to include(:headers) + expect(request.options[:headers]).to include('Authorization' => auth) + end end end From 902fbd1fafbd0f65b2715aa15fd66b155d907c09 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 4 Mar 2021 07:57:13 -0500 Subject: [PATCH 066/289] Explicitly state type when retrieving cached URLs --- lib/html-proofer/cache.rb | 33 ++++++-- .../html-proofer/fixtures/cache/.external.log | 2 +- .../fixtures/cache/.internal_and_external.log | 2 +- .../fixtures/cache/.removed_link.log | 2 +- spec/html-proofer/fixtures/cache/.runner.log | 1 + .../links/_site/folder.html/index.html | 6 +- .../cache_log_level_error_type_links_.yml | 81 ++++++++++++++++++ ...er_log_log_level_error_type_directory_.yml | 83 +++++++++++++++++++ .../cache_log_level_error_type_directory_.yml | 83 +++++++++++++++++++ ...ame_1d_log_level_error_type_directory_.yml | 83 +++++++++++++++++++ ..._once_html_log_level_error_type_links_.yml | 82 ++++++++++++++++++ .../cache_log_level_error_type_file_.yml | 76 +++++++++++++++++ 12 files changed, 521 insertions(+), 13 deletions(-) create mode 100644 spec/html-proofer/fixtures/cache/.runner.log create mode 100644 spec/html-proofer/fixtures/vcr_cassettes/https_/www_github_com_cache_timeframe_4d_cache_file_within_date_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml create mode 100644 spec/html-proofer/fixtures/vcr_cassettes/links/_site_cache_timeframe_1d_cache_file_runner_log_log_level_error_type_directory_.yml create mode 100644 spec/html-proofer/fixtures/vcr_cassettes/links/_site_cache_timeframe_1d_cache_file_runner_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_directory_.yml create mode 100644 spec/html-proofer/fixtures/vcr_cassettes/links/_site_cache_timeframe_1d_log_level_error_type_directory_.yml create mode 100644 spec/html-proofer/fixtures/vcr_cassettes/links/check_just_once_html_log_level_error_type_links_.yml create mode 100644 spec/html-proofer/fixtures/vcr_cassettes/links/missing_link_href_html_cache_timeframe_30d_cache_file_htmlproofer_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml diff --git a/lib/html-proofer/cache.rb b/lib/html-proofer/cache.rb index 9048588b..768a308d 100644 --- a/lib/html-proofer/cache.rb +++ b/lib/html-proofer/cache.rb @@ -11,6 +11,8 @@ class Cache DEFAULT_STORAGE_DIR = File.join('tmp', '.htmlproofer') DEFAULT_CACHE_FILE_NAME = 'cache.log' + URI_REGEXP = URI::DEFAULT_PARSER.make_regexp + attr_reader :exists, :cache_log, :storage_dir, :cache_file def initialize(logger, options) @@ -30,6 +32,8 @@ def initialize(logger, options) end def within_timeframe?(time) + return false if time.nil? + (@parsed_timeframe..@cache_time).cover?(Time.parse(time)) end @@ -72,9 +76,13 @@ def add(url, filenames, status, msg = '') end def detect_url_changes(found, type) - existing_urls = @cache_log.keys.map { |url| clean_url(url) } found_urls = found.keys.map { |url| clean_url(url) } + # if there were no urls, bail + return {} if found_urls.empty? + + existing_urls = @cache_log.keys.map { |url| clean_url(url) } + # prepare to add new URLs detected additions = found.reject do |url, _| url = clean_url(url) @@ -91,21 +99,20 @@ def detect_url_changes(found, type) @logger.log :info, "Adding #{new_link_text} to the cache..." # remove from cache URLs that no longer exist - del = 0 - uri_regexp = URI::DEFAULT_PARSER.make_regexp + deletions = 0 @cache_log.delete_if do |url, _| url = clean_url(url) if found_urls.include?(url) false - elsif (type == :internal && url !~ uri_regexp) || (type == :external && url =~ uri_regexp) + elsif url_matches_type?(url, type) @logger.log :debug, "Removing #{url} from cache check" - del += 1 + deletions += 1 true end end - del_link_text = pluralize(del, 'link', 'links') + del_link_text = pluralize(deletions, 'link', 'links') @logger.log :info, "Removing #{del_link_text} from the cache..." additions @@ -124,10 +131,13 @@ def load? def retrieve_urls(urls, type) urls_to_check = detect_url_changes(urls, type) + @cache_log.each_pair do |url, cache| next if within_timeframe?(cache['time']) && cache['message'].empty? # these were successes to skip - urls_to_check[url] = cache['filenames'] # recheck expired links + if url_matches_type?(url, type) + urls_to_check[url] = cache['filenames'] # recheck expired links + end end urls_to_check end @@ -154,9 +164,9 @@ def setup_cache!(options) @cache_file = File.join(storage_dir, cache_file_name) - return unless File.exist?(cache_file) + return unless File.exist?(@cache_file) - contents = File.read(cache_file) + contents = File.read(@cache_file) @cache_log = contents.empty? ? {} : JSON.parse(contents) end @@ -174,5 +184,10 @@ def time_ago(measurement, unit) @cache_datetime - Rational(measurement / 24.0) end.to_time end + + def url_matches_type?(url, type) + return true if type == :internal && url !~ URI_REGEXP + return true if type == :external && url =~ URI_REGEXP + end end end diff --git a/spec/html-proofer/fixtures/cache/.external.log b/spec/html-proofer/fixtures/cache/.external.log index d147adeb..e434d244 100644 --- a/spec/html-proofer/fixtures/cache/.external.log +++ b/spec/html-proofer/fixtures/cache/.external.log @@ -1 +1 @@ -{"https://github.com/gjtorikian/html-proofer":{"time":"2021-01-27 12:00:00 +0000","filenames":["spec/html-proofer/fixtures/cache/external_example.html"],"status":200,"message":""}} \ No newline at end of file +{"https://github.com/gjtorikian/html-proofer":{"time":"2021-01-27 12:00:00 -0500","filenames":["spec/html-proofer/fixtures/cache/external_example.html"],"status":200,"message":""}} \ No newline at end of file diff --git a/spec/html-proofer/fixtures/cache/.internal_and_external.log b/spec/html-proofer/fixtures/cache/.internal_and_external.log index 17c57ce2..7b2898a9 100644 --- a/spec/html-proofer/fixtures/cache/.internal_and_external.log +++ b/spec/html-proofer/fixtures/cache/.internal_and_external.log @@ -1 +1 @@ -{"https://github.com/gjtorikian/html-proofer":{"time":"2021-01-27 12:00:00 +0000","filenames":["spec/html-proofer/fixtures/cache/internal_and_external_example.html"],"status":200,"message":""},"/somewhere.html":{"time":"2021-01-27 12:00:00 +0000","filenames":["spec/html-proofer/fixtures/cache/internal_and_external_example.html"],"status":200,"message":""}} \ No newline at end of file +{"https://github.com/gjtorikian/html-proofer":{"time":"2021-01-27 12:00:00 -0500","filenames":["spec/html-proofer/fixtures/cache/internal_and_external_example.html"],"status":200,"message":""},"/somewhere.html":{"time":"2021-01-27 12:00:00 -0500","filenames":["spec/html-proofer/fixtures/cache/internal_and_external_example.html"],"status":200,"message":""}} \ No newline at end of file diff --git a/spec/html-proofer/fixtures/cache/.removed_link.log b/spec/html-proofer/fixtures/cache/.removed_link.log index 7de644c8..00bb8f0e 100644 --- a/spec/html-proofer/fixtures/cache/.removed_link.log +++ b/spec/html-proofer/fixtures/cache/.removed_link.log @@ -1 +1 @@ -{"https://github.com/gjtorikian":{"time":"2021-01-27 12:00:00 +0000","filenames":["spec/html-proofer/fixtures/cache/some_link.html"],"status":200,"message":""}} \ No newline at end of file +{"https://github.com/gjtorikian":{"time":"2021-01-27 12:00:00 -0500","filenames":["spec/html-proofer/fixtures/cache/some_link.html"],"status":200,"message":""}} \ No newline at end of file diff --git a/spec/html-proofer/fixtures/cache/.runner.log b/spec/html-proofer/fixtures/cache/.runner.log new file mode 100644 index 00000000..9811a22c --- /dev/null +++ b/spec/html-proofer/fixtures/cache/.runner.log @@ -0,0 +1 @@ +{"https://www.github.com":{"time":"2021-03-04 07:54:18 -0500","filenames":["spec/html-proofer/fixtures/links/_site/folder.html/index.html"],"status":200,"message":""}} \ No newline at end of file diff --git a/spec/html-proofer/fixtures/links/_site/folder.html/index.html b/spec/html-proofer/fixtures/links/_site/folder.html/index.html index fad1f47f..73e0bc72 100644 --- a/spec/html-proofer/fixtures/links/_site/folder.html/index.html +++ b/spec/html-proofer/fixtures/links/_site/folder.html/index.html @@ -1 +1,5 @@ -Dummy + + + Dummy + diff --git a/spec/html-proofer/fixtures/vcr_cassettes/https_/www_github_com_cache_timeframe_4d_cache_file_within_date_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml b/spec/html-proofer/fixtures/vcr_cassettes/https_/www_github_com_cache_timeframe_4d_cache_file_within_date_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml new file mode 100644 index 00000000..7bdfe5b9 --- /dev/null +++ b/spec/html-proofer/fixtures/vcr_cassettes/https_/www_github_com_cache_timeframe_4d_cache_file_within_date_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_links_.yml @@ -0,0 +1,81 @@ +--- +http_interactions: +- request: + method: head + uri: https://www.github.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.6; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + server: + - GitHub.com + date: + - Thu, 04 Mar 2021 12:52:54 GMT + content-type: + - text/html; charset=utf-8 + vary: + - X-PJAX, Accept-Language, Accept-Encoding, Accept, X-Requested-With + x-rails-requested-accept-language: + - en + content-language: + - en-US + etag: + - W/"f3c636bf3c61aa81b2ab51c190efc998" + cache-control: + - max-age=0, private, must-revalidate + strict-transport-security: + - max-age=31536000; includeSubdomains; preload + x-frame-options: + - deny + x-content-type-options: + - nosniff + x-xss-protection: + - 1; mode=block + referrer-policy: + - origin-when-cross-origin, strict-origin-when-cross-origin + expect-ct: + - max-age=2592000, report-uri="https://api.github.com/_private/browser/errors" + content-security-policy: + - 'default-src ''none''; base-uri ''self''; block-all-mixed-content; connect-src + ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com + github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com + github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com + cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com github.githubassets.com; + font-src github.githubassets.com; form-action ''self'' github.com gist.github.com; + frame-ancestors ''none''; frame-src render.githubusercontent.com; img-src + ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com + github-cloud.s3.amazonaws.com user-images.githubusercontent.com/ *.githubusercontent.com + customer-stories-feed.github.com spotlights-feed.github.com; manifest-src + ''self''; media-src github.githubassets.com; script-src github.githubassets.com; + style-src ''unsafe-inline'' github.githubassets.com; worker-src github.com/socket-worker-5029ae85.js + gist.github.com/socket-worker-5029ae85.js' + set-cookie: + - _gh_sess=WnbixycLsdz6Orn4f0sI6FYNghW7jPu0fJb6yT88AwHe7W5nKh%2BYGYk9YeDzsFL7chui3R9s0ynvwbg0qklzP3TkM95zVuJPE0%2Fdl6WaFXAAyV%2BrKlyCD%2FFuJiz7xNXe0NSS5%2BMlzvwG7JIkGqwG07MUq8S85BlPfDdxAwTPhkQzUxJ53RU0tLU0PV8JLLzVn%2B4JHedhJxLzXE6g7YJ8pmPcXHVsk51b%2Fow1OyCLx8Ffthg%2Bs6YVGkaf1hD2XjwOCMhtv%2BkqqRRqsU3tnqRgNQ%3D%3D--271NITNBCDNj1avQ--jDbWSymyZw63SkAvnrZMkw%3D%3D; + Path=/; HttpOnly; Secure; SameSite=Lax + - _octo=GH1.1.1578498300.1614862374; Path=/; Domain=github.com; Expires=Fri, + 04 Mar 2022 12:52:54 GMT; Secure; SameSite=Lax + - logged_in=no; Path=/; Domain=github.com; Expires=Fri, 04 Mar 2022 12:52:54 + GMT; HttpOnly; Secure; SameSite=Lax + accept-ranges: + - bytes + x-github-request-id: + - B8D9:3C0C:5C1B36:B765B1:6040D826 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://github.com/ + recorded_at: Tue, 27 Oct 2015 16:00:00 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html-proofer/fixtures/vcr_cassettes/links/_site_cache_timeframe_1d_cache_file_runner_log_log_level_error_type_directory_.yml b/spec/html-proofer/fixtures/vcr_cassettes/links/_site_cache_timeframe_1d_cache_file_runner_log_log_level_error_type_directory_.yml new file mode 100644 index 00000000..fab20bfc --- /dev/null +++ b/spec/html-proofer/fixtures/vcr_cassettes/links/_site_cache_timeframe_1d_cache_file_runner_log_log_level_error_type_directory_.yml @@ -0,0 +1,83 @@ +--- +http_interactions: +- request: + method: head + uri: https://www.github.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.6; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + Authorization: + - Bearer + response: + status: + code: 200 + message: '' + headers: + server: + - GitHub.com + date: + - Thu, 04 Mar 2021 12:21:21 GMT + content-type: + - text/html; charset=utf-8 + vary: + - X-PJAX, Accept-Language, Accept-Encoding, Accept, X-Requested-With + x-rails-requested-accept-language: + - en + content-language: + - en-US + etag: + - W/"6706bb34bf7f0d382798736ca7735dec" + cache-control: + - max-age=0, private, must-revalidate + strict-transport-security: + - max-age=31536000; includeSubdomains; preload + x-frame-options: + - deny + x-content-type-options: + - nosniff + x-xss-protection: + - 1; mode=block + referrer-policy: + - origin-when-cross-origin, strict-origin-when-cross-origin + expect-ct: + - max-age=2592000, report-uri="https://api.github.com/_private/browser/errors" + content-security-policy: + - 'default-src ''none''; base-uri ''self''; block-all-mixed-content; connect-src + ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com + github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com + github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com + cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com github.githubassets.com; + font-src github.githubassets.com; form-action ''self'' github.com gist.github.com; + frame-ancestors ''none''; frame-src render.githubusercontent.com; img-src + ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com + github-cloud.s3.amazonaws.com user-images.githubusercontent.com/ *.githubusercontent.com + customer-stories-feed.github.com spotlights-feed.github.com; manifest-src + ''self''; media-src github.githubassets.com; script-src github.githubassets.com; + style-src ''unsafe-inline'' github.githubassets.com; worker-src github.com/socket-worker-5029ae85.js + gist.github.com/socket-worker-5029ae85.js' + set-cookie: + - _gh_sess=UYfovP46kNIH8yQsnEnZ9OisXWbAramt4olNToj4xiFUycY3Tf5Gf0iNT7Qxpd1jF4OIsRxUaPTUQt01dpJdRAhMyOcCedHuNthXIhZdFszpOzUDjRjCGXsGH8VucAGkn2IIOTNJXVKdGWrRyvEwkHT243doL2mXA86nDGlO8cR9%2FxcOHDCNizyy4%2BX4kK%2BD7SK9KdWBCI8dC7PyCH%2BuAc6tcWjOJCSg7Wv6XEBG5OFmRSEnlwiOu2uK2nQzmNi7gHPcvU%2FFiYSOFF5knwCuQA%3D%3D--85LqMaS3k4xfIwDM--3smwH%2B2UQQ%2B5HRuqbP0M4w%3D%3D; + Path=/; HttpOnly; Secure; SameSite=Lax + - _octo=GH1.1.760296063.1614860481; Path=/; Domain=github.com; Expires=Fri, + 04 Mar 2022 12:21:21 GMT; Secure; SameSite=Lax + - logged_in=no; Path=/; Domain=github.com; Expires=Fri, 04 Mar 2022 12:21:21 + GMT; HttpOnly; Secure; SameSite=Lax + accept-ranges: + - bytes + x-github-request-id: + - 9858:06BA:B7812C:12C2BD2:6040D0C1 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://github.com/ + recorded_at: Thu, 04 Mar 2021 12:21:21 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html-proofer/fixtures/vcr_cassettes/links/_site_cache_timeframe_1d_cache_file_runner_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_directory_.yml b/spec/html-proofer/fixtures/vcr_cassettes/links/_site_cache_timeframe_1d_cache_file_runner_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_directory_.yml new file mode 100644 index 00000000..34e0d00e --- /dev/null +++ b/spec/html-proofer/fixtures/vcr_cassettes/links/_site_cache_timeframe_1d_cache_file_runner_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_directory_.yml @@ -0,0 +1,83 @@ +--- +http_interactions: +- request: + method: head + uri: https://www.github.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.6; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + Authorization: + - Bearer + response: + status: + code: 200 + message: '' + headers: + server: + - GitHub.com + date: + - Thu, 04 Mar 2021 12:23:48 GMT + content-type: + - text/html; charset=utf-8 + vary: + - X-PJAX, Accept-Language, Accept-Encoding, Accept, X-Requested-With + x-rails-requested-accept-language: + - en + content-language: + - en-US + etag: + - W/"c96a206715c17bc47fee33588c752878" + cache-control: + - max-age=0, private, must-revalidate + strict-transport-security: + - max-age=31536000; includeSubdomains; preload + x-frame-options: + - deny + x-content-type-options: + - nosniff + x-xss-protection: + - 1; mode=block + referrer-policy: + - origin-when-cross-origin, strict-origin-when-cross-origin + expect-ct: + - max-age=2592000, report-uri="https://api.github.com/_private/browser/errors" + content-security-policy: + - 'default-src ''none''; base-uri ''self''; block-all-mixed-content; connect-src + ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com + github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com + github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com + cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com github.githubassets.com; + font-src github.githubassets.com; form-action ''self'' github.com gist.github.com; + frame-ancestors ''none''; frame-src render.githubusercontent.com; img-src + ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com + github-cloud.s3.amazonaws.com user-images.githubusercontent.com/ *.githubusercontent.com + customer-stories-feed.github.com spotlights-feed.github.com; manifest-src + ''self''; media-src github.githubassets.com; script-src github.githubassets.com; + style-src ''unsafe-inline'' github.githubassets.com; worker-src github.com/socket-worker-5029ae85.js + gist.github.com/socket-worker-5029ae85.js' + set-cookie: + - _gh_sess=mOWa3zWxCYK4TBGYFaSfsKyYrqvKo73YAmQzP64CcKx1BKMJVpn7GdcUqqsSGbmMyHxVu8PJosCv763roz%2BKWdTwCvmAzsUOQxD44UfJbmZV1XF2msNYSJj2RmoLs4aC4SLS2548i2PYJh%2BzOH6INPykajYU02lcaHgX%2BhYUcQSV2pG8funqq9kQT%2Fkbv6PfqZ8o08de56u%2Feg2ibGPhRGy49GakkDYILo1Zufl8wKtdBoQsViTXYMuo%2F3IUAIyeOADOMtgHntSoFdQ8dqoQMA%3D%3D--f3QuqBSZiV%2BHBihX--ruRV330LLsoKH0tWPr4k5w%3D%3D; + Path=/; HttpOnly; Secure; SameSite=Lax + - _octo=GH1.1.1369297613.1614860628; Path=/; Domain=github.com; Expires=Fri, + 04 Mar 2022 12:23:48 GMT; Secure; SameSite=Lax + - logged_in=no; Path=/; Domain=github.com; Expires=Fri, 04 Mar 2022 12:23:48 + GMT; HttpOnly; Secure; SameSite=Lax + accept-ranges: + - bytes + x-github-request-id: + - 9922:6058:2F1210:7554D3:6040D154 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://github.com/ + recorded_at: Thu, 04 Mar 2021 12:23:48 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html-proofer/fixtures/vcr_cassettes/links/_site_cache_timeframe_1d_log_level_error_type_directory_.yml b/spec/html-proofer/fixtures/vcr_cassettes/links/_site_cache_timeframe_1d_log_level_error_type_directory_.yml new file mode 100644 index 00000000..6c8b6aa1 --- /dev/null +++ b/spec/html-proofer/fixtures/vcr_cassettes/links/_site_cache_timeframe_1d_log_level_error_type_directory_.yml @@ -0,0 +1,83 @@ +--- +http_interactions: +- request: + method: head + uri: https://www.github.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.6; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + Authorization: + - Bearer + response: + status: + code: 200 + message: '' + headers: + server: + - GitHub.com + date: + - Thu, 04 Mar 2021 12:17:38 GMT + content-type: + - text/html; charset=utf-8 + vary: + - X-PJAX, Accept-Language, Accept-Encoding, Accept, X-Requested-With + x-rails-requested-accept-language: + - en + content-language: + - en-US + etag: + - W/"280eb5a9580dfaa5550f70a65fb8188e" + cache-control: + - max-age=0, private, must-revalidate + strict-transport-security: + - max-age=31536000; includeSubdomains; preload + x-frame-options: + - deny + x-content-type-options: + - nosniff + x-xss-protection: + - 1; mode=block + referrer-policy: + - origin-when-cross-origin, strict-origin-when-cross-origin + expect-ct: + - max-age=2592000, report-uri="https://api.github.com/_private/browser/errors" + content-security-policy: + - 'default-src ''none''; base-uri ''self''; block-all-mixed-content; connect-src + ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com + github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com + github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com + cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com github.githubassets.com; + font-src github.githubassets.com; form-action ''self'' github.com gist.github.com; + frame-ancestors ''none''; frame-src render.githubusercontent.com; img-src + ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com + github-cloud.s3.amazonaws.com user-images.githubusercontent.com/ *.githubusercontent.com + customer-stories-feed.github.com spotlights-feed.github.com; manifest-src + ''self''; media-src github.githubassets.com; script-src github.githubassets.com; + style-src ''unsafe-inline'' github.githubassets.com; worker-src github.com/socket-worker-5029ae85.js + gist.github.com/socket-worker-5029ae85.js' + set-cookie: + - _gh_sess=vFAcRFr%2BiTygmwsGhDrge1iFqt2ICZuA%2FbZehUmVsVBsQMneP%2Fv7PDO4PnQ9UbBScDP%2BZ98n4ijrqQuh1EDtbdGdrJ0Zc%2BWM1JL%2BQHtJg5cFHJLS73l4k2t2nh%2BEC7DsoMvKEXyPazbx71TMtXSh%2F23HuFX7j9XKIwuVEXZbluyiZItONsOMd2Katttjpy8zXKmFD8bI1Blppx1mkYy%2BVYpyEDHHBbn5Qe27BV5qe9zgSzwiG%2Ftu50mC2jR4jiZOyfDVjZhO7CeovrqdZbmFQQ%3D%3D--txfx90gYFeVygoiJ--NogIeXYdKOdEu5dCBMKnZw%3D%3D; + Path=/; HttpOnly; Secure; SameSite=Lax + - _octo=GH1.1.1648158114.1614860258; Path=/; Domain=github.com; Expires=Fri, + 04 Mar 2022 12:17:38 GMT; Secure; SameSite=Lax + - logged_in=no; Path=/; Domain=github.com; Expires=Fri, 04 Mar 2022 12:17:38 + GMT; HttpOnly; Secure; SameSite=Lax + accept-ranges: + - bytes + x-github-request-id: + - A620:42A5:412094:8A0932:6040CFE2 + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://github.com/ + recorded_at: Thu, 04 Mar 2021 12:17:38 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html-proofer/fixtures/vcr_cassettes/links/check_just_once_html_log_level_error_type_links_.yml b/spec/html-proofer/fixtures/vcr_cassettes/links/check_just_once_html_log_level_error_type_links_.yml new file mode 100644 index 00000000..eded7918 --- /dev/null +++ b/spec/html-proofer/fixtures/vcr_cassettes/links/check_just_once_html_log_level_error_type_links_.yml @@ -0,0 +1,82 @@ +--- +http_interactions: +- request: + method: head + uri: https://github.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.6; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + Authorization: + - Bearer + response: + status: + code: 200 + message: '' + headers: + server: + - GitHub.com + date: + - Thu, 04 Mar 2021 12:15:23 GMT + content-type: + - text/html; charset=utf-8 + vary: + - X-PJAX, Accept-Language + - Accept-Encoding, Accept, X-Requested-With + x-rails-requested-accept-language: + - '' + content-language: + - en-US + etag: + - W/"950d1ceb8388b3a5d43171d9f9460629" + cache-control: + - max-age=0, private, must-revalidate + set-cookie: + - _octo=GH1.1.1274787104.1614860123; domain=.github.com; path=/; expires=Fri, + 04 Mar 2022 12:15:23 GMT; secure; SameSite=Lax + - logged_in=no; domain=.github.com; path=/; expires=Fri, 04 Mar 2022 12:15:23 + GMT; secure; HttpOnly; SameSite=Lax + - _gh_sess=aSp1g1TcH%2Fxlf26wZJq0u4%2B1nuvt2fWEiRaLMsxDBczBksykrHsSQ8s4R8hSJY9PAytgZGLf4AVciGdketbIxmKK9uiuAhjPlnStct3o0D1HuBEYbc0q1a2UDR07cZqaRDkMdSUird2l1dTsmZKyk03ChrCMJDEXZQYF8GyYMUL6G%2FxoVw5f6ACHaqyDmYKBYwmxHcNXBAsYLfLMqky6U04nRLkah3ZRYywaVH0vW8pDUC2WxJCVuveS9iV2LsbCbC6JNCg2aKHmz2sTHKyfJQ%3D%3D--edbrk6MdnYzjDcDD--4Atgx8pVUOKGHVG2zRDrJw%3D%3D; + path=/; secure; HttpOnly; SameSite=Lax + strict-transport-security: + - max-age=31536000; includeSubdomains; preload + x-frame-options: + - deny + x-content-type-options: + - nosniff + x-xss-protection: + - 1; mode=block + referrer-policy: + - origin-when-cross-origin, strict-origin-when-cross-origin + expect-ct: + - max-age=2592000, report-uri="https://api.github.com/_private/browser/errors" + content-security-policy: + - 'default-src ''none''; base-uri ''self''; block-all-mixed-content; connect-src + ''self'' uploads.github.com www.githubstatus.com collector.githubapp.com api.github.com + github-cloud.s3.amazonaws.com github-production-repository-file-5c1aeb.s3.amazonaws.com + github-production-upload-manifest-file-7fdce7.s3.amazonaws.com github-production-user-asset-6210df.s3.amazonaws.com + cdn.optimizely.com logx.optimizely.com/v1/events wss://alive.github.com github.githubassets.com; + font-src github.githubassets.com; form-action ''self'' github.com gist.github.com; + frame-ancestors ''none''; frame-src render.githubusercontent.com; img-src + ''self'' data: github.githubassets.com identicons.github.com collector.githubapp.com + github-cloud.s3.amazonaws.com user-images.githubusercontent.com/ *.githubusercontent.com + customer-stories-feed.github.com spotlights-feed.github.com; manifest-src + ''self''; media-src github.githubassets.com; script-src github.githubassets.com; + style-src ''unsafe-inline'' github.githubassets.com; worker-src github.com/socket-worker-5029ae85.js + gist.github.com/socket-worker-5029ae85.js' + x-github-request-id: + - 8C57:68C1:F995FA:17DD73D:6040CF5B + body: + encoding: UTF-8 + string: '' + http_version: '2' + adapter_metadata: + effective_url: https://github.com + recorded_at: Thu, 04 Mar 2021 12:15:23 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html-proofer/fixtures/vcr_cassettes/links/missing_link_href_html_cache_timeframe_30d_cache_file_htmlproofer_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml b/spec/html-proofer/fixtures/vcr_cassettes/links/missing_link_href_html_cache_timeframe_30d_cache_file_htmlproofer_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml new file mode 100644 index 00000000..0e816482 --- /dev/null +++ b/spec/html-proofer/fixtures/vcr_cassettes/links/missing_link_href_html_cache_timeframe_30d_cache_file_htmlproofer_log_storage_dir_spec/html-proofer/fixtures/cache_log_level_error_type_file_.yml @@ -0,0 +1,76 @@ +--- +http_interactions: +- request: + method: head + uri: http://www.asdo3irj395295jsingrkrg4.com/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.6; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: http://www.asdo3irj395295jsingrkrg4.com/ + recorded_at: Thu, 04 Mar 2021 12:36:32 GMT +- request: + method: head + uri: http://www.google.com/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/3.18.6; +https://github.com/gjtorikian/html-proofer) + Accept: + - application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5 + Expect: + - '' + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - text/html; charset=UTF-8 + P3P: + - CP="This is not a P3P policy! See g.co/p3phelp for more info." + Date: + - Thu, 04 Mar 2021 12:36:32 GMT + Server: + - gws + X-XSS-Protection: + - '0' + X-Frame-Options: + - SAMEORIGIN + Transfer-Encoding: + - chunked + Expires: + - Thu, 04 Mar 2021 12:36:32 GMT + Cache-Control: + - private + Set-Cookie: + - 1P_JAR=2021-03-04-12; expires=Sat, 03-Apr-2021 12:36:32 GMT; path=/; domain=.google.com; + Secure + - NID=210=JqMOj0JuTOvaL7xHIVQYDGZZbMv9cr0nO1s-s7gfnjYKnhgAeo2Pa6XXKnOjbDiij3IAknU86hBSSYTb--N7rMldYjTjzm6y23RLM3zqT6CKTzVAw_MYc_tsAoEa5j_7tOIMCUcknCZOjB6StJ5GYd4AKTF_j_vNR5pfbrnnC68; + expires=Fri, 03-Sep-2021 12:36:32 GMT; path=/; domain=.google.com; HttpOnly + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: http://www.google.com/ + recorded_at: Thu, 04 Mar 2021 12:36:32 GMT +recorded_with: VCR 2.9.3 From 5d03743315fcacf5122e52106a57d2689fdf7a9b Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 4 Mar 2021 08:00:56 -0500 Subject: [PATCH 067/289] :gem: bump to 3.18.7 --- lib/html-proofer/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/version.rb b/lib/html-proofer/version.rb index f0a75c53..3eec7f78 100644 --- a/lib/html-proofer/version.rb +++ b/lib/html-proofer/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HTMLProofer - VERSION = '3.18.6' + VERSION = '3.18.7' end From e1c7d08264b0d597d8e3e9476706318808079a58 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 4 Mar 2021 12:05:51 -0500 Subject: [PATCH 068/289] require --- lib/html-proofer/cache.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/html-proofer/cache.rb b/lib/html-proofer/cache.rb index 768a308d..e56be767 100644 --- a/lib/html-proofer/cache.rb +++ b/lib/html-proofer/cache.rb @@ -3,6 +3,7 @@ require_relative 'utils' require 'date' require 'json' +require 'uri' module HTMLProofer class Cache From 8d606b011facd2eebc3e8e3d9b9ac00faa99e959 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Thu, 4 Mar 2021 12:06:07 -0500 Subject: [PATCH 069/289] :gem: bump to 3.18.8 --- lib/html-proofer/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/html-proofer/version.rb b/lib/html-proofer/version.rb index 3eec7f78..ceecb8a9 100644 --- a/lib/html-proofer/version.rb +++ b/lib/html-proofer/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module HTMLProofer - VERSION = '3.18.7' + VERSION = '3.18.8' end From 101059f9ab7d79769091f347c4124de43f6b3ed9 Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Tue, 9 Mar 2021 07:28:55 -0500 Subject: [PATCH 070/289] let push through --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d519718d..89a43db8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,8 +4,6 @@ on: pull_request: types: [opened, synchronize, reopened] push: - branches: - - main jobs: test: From 1707bd72bdebdd7c495f800abdf5e8454c941aac Mon Sep 17 00:00:00 2001 From: Patrick Connolly Date: Mon, 29 Mar 2021 16:13:56 -0400 Subject: [PATCH 071/289] Added --hydra-config flag to override concurrency from CLI. --- bin/htmlproofer | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/htmlproofer b/bin/htmlproofer index 8c4c3362..9d95e4e9 100755 --- a/bin/htmlproofer +++ b/bin/htmlproofer @@ -48,6 +48,7 @@ Mercenary.program(:htmlproofer) do |p| p.option 'storage_dir', '--storage-dir PATH', String, 'Directory where to store the cache log (default: "tmp/.htmlproofer")' p.option 'timeframe', '--timeframe