From b186d7ae68c6e4e6f9b891291c3e29242e6ced0a Mon Sep 17 00:00:00 2001
From: myfreeer <myfreeer@users.noreply.github.com>
Date: Sat, 5 Dec 2020 18:52:51 +0800
Subject: [PATCH] mdn: skip incorrectly parsed url

---
 src/mdn/process-url/skip-process.ts       | 18 +++++++++++++++---
 test/mdn/process-url/skip-process.spec.ts |  5 +++++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/mdn/process-url/skip-process.ts b/src/mdn/process-url/skip-process.ts
index 4dcb08b1..907b0913 100644
--- a/src/mdn/process-url/skip-process.ts
+++ b/src/mdn/process-url/skip-process.ts
@@ -1,4 +1,7 @@
-import {skipExternal as skipExternalLogger} from 'website-scrap-engine/lib/logger/logger';
+import {
+  skipExternal as skipExternalLogger,
+  error as errorLogger
+} from 'website-scrap-engine/lib/logger/logger';
 import URI from 'urijs';
 import { downloadableHosts } from './consts';
 import {Resource} from 'website-scrap-engine/lib/resource';
@@ -42,9 +45,18 @@ export const skipProcess = (
   }
   const uri = URI(url), host = uri.host();
   if (host && !downloadableHosts[host]) {
-    skipExternalLogger.debug('skipped external link', host, url, parent && parent.url);
+    skipExternalLogger.debug('skipped external link', host, url, parent?.url);
     return;
   }
+  // incorrectly parsed url
+  // localhost:3000
+  // from https://developer.mozilla.org/zh-CN/docs/Learn/Tools_and_testing/
+  // Client-side_JavaScript_frameworks/React_getting_started
+  if (uri.is('absolute') && !host) {
+    errorLogger.info('skipped incorrectly parsed url', url, parent?.url);
+    return;
+  }
+
   const path = uri.path();
   if (path.startsWith('/presentations/') ||
     // very large file
@@ -54,7 +66,7 @@ export const skipProcess = (
     path.startsWith('/files/5243/IconsCommunications_20130401.psd') ||
     path.startsWith('/files/5245/IconsSettings_20130415.psd') ||
     path.startsWith('/files/5247/IconsPrimaryAction_20130501.psd')) {
-    skipExternalLogger.debug('skipped link to large file', url, parent && parent.url);
+    skipExternalLogger.debug('skipped link to large file', url, parent?.url);
     return;
   }
   return url;
diff --git a/test/mdn/process-url/skip-process.spec.ts b/test/mdn/process-url/skip-process.spec.ts
index 96ff1709..0b59a49b 100644
--- a/test/mdn/process-url/skip-process.spec.ts
+++ b/test/mdn/process-url/skip-process.spec.ts
@@ -60,4 +60,9 @@ describe('skip-process', function () {
     expect(skipProcess('http://www.mozilla.org/favicon.ico', null, null))
       .toBe('https://developer.mozilla.org/static/img/favicon32.png');
   });
+
+  test('skip incorrectly-parsed url', () => {
+    // https://developer.mozilla.org/zh-CN/docs/Learn/Tools_and_testing/Client-side_JavaScript_frameworks/React_getting_started
+    exceptSkipped('localhost:3000');
+  });
 });