[pull] main from xnl-h4ck3r:main

CHANGELOG.md

@@ -1,5 +1,16 @@
 ## Changelog
 
+- v3.8
+
+  - New
+
+    - When argument `-vv` is passed, display the Content-Types that were processed for links and parameters. This can be used to identify any obscure content types that you would want to exclude in future, so that can be manually added to the `contentExclude` section of `config.yml` if required.
+
+  - Changed
+
+    - Add these content types to the `DEFAULT_CONTENTTYPE_EXCLUSIONS` constant, and the `contentExclude` section of `config.yml`: `application/zip,application/x-zip-compressed,application/x-msdownload,application/x-apple-diskimage,application/x-rpm,application/vnd.debian.binary-package`
+    - Check the File Extension exclusions for URLs too, if a content type wasn't found
+
 - v3.7
 
   - Changed

README.md

@@ -1,6 +1,6 @@
 <center><img src="https://github.com/xnl-h4ck3r/xnLinkFinder/blob/main/xnLinkFinder/images/title.png"></center>
 
-## About - v3.7
+## About - v3.8
 
 This is a tool used to discover endpoints (and potential parameters) for a given target. It can find them by:
 
@@ -81,7 +81,7 @@ The `config.yml` file has the keys which can be updated to suit your needs:
 
 - `linkExclude` - A comma separated list of strings (e.g. `.css,.jpg,.jpeg` etc.) that all links are checked against. If a link includes any of the strings then it will be excluded from the output. If the input is a directory, then file names are checked against this list.
 - `contentExclude` - A comma separated list of strings (e.g. `text/css,image/jpeg,image/jpg` etc.) that all responses `Content-Type` headers are checked against. Any responses with the these content types will be excluded and not checked for links.
-- `fileExtExclude` - A comma separated list of strings (e.g. `.zip,.gz,.tar` etc.) that all files in Directory mode are checked against. If a file has one of those extensions it will not be searched for links.
+- `fileExtExclude` - A comma separated list of strings (e.g. `.zip,.gz,.tar` etc.) that all files in Directory mode are checked against. If a file has one of those extensions it will not be searched for links. Also, in normal mode, if a response doesn't have a content-type to check for exclusions, it will check for these extensions at the end of the URL to determine if to search for links.
 - `regexFiles` - A list of file types separated by a pipe character (e.g. `php|php3|php5` etc.). These are used in the Link Finding Regex when there are findings that aren't obvious links, but are interesting file types that you want to pick out. If you add to this list, ensure you escape any dots to ensure correct regex, e.g. `js\.map`
 - `respParamLinksFound` † - Whether to get potential parameters from links found in responses: `True` or `False`
 - `respParamPathWords` † - Whether to add path words in retrieved links as potential parameters: `True` or `False`

config.yml

@@ -1,5 +1,5 @@
 linkExclude: .css,.jpg,.jpeg,.png,.svg,.img,.gif,.mp4,.flv,.ogv,.webm,.webp,.mov,.mp3,.m4a,.m4p,.scss,.tif,.tiff,.ttf,.otf,.woff,.woff2,.bmp,.ico,.eot,.htc,.rtf,.swf,.image,w3.org,doubleclick.net,youtube.com,.vue,jquery,bootstrap,font,jsdelivr.net,vimeo.com,pinterest.com,facebook,linkedin,twitter,instagram,google,mozilla.org,jibe.com,schema.org,schemas.microsoft.com,wordpress.org,w.org,wix.com,parastorage.com,whatwg.org,polyfill.io,typekit.net,schemas.openxmlformats.org,openweathermap.org,openoffice.org,reactjs.org,angularjs.org,java.com,purl.org,/image,/img,/css,/wp-json,/wp-content,/wp-includes,/theme,/audio,/captcha,/font,robots.txt,node_modules,.wav,.gltf,.pict,.svgz,.eps,.midi,.mid,.avif
-contentExclude: text/css,image/jpeg,image/jpg,image/png,image/svg+xml,image/gif,image/tiff,image/webp,image/bmp,image/x-icon,image/vnd.microsoft.icon,font/ttf,font/woff,font/woff2,font/x-woff2,font/x-woff,font/otf,audio/mpeg,audio/wav,audio/webm,audio/aac,audio/ogg,audio/wav,audio/webm,video/mp4,video/mpeg,video/webm,video/ogg,video/mp2t,video/webm,video/x-msvideo,application/font-woff,application/font-woff2,application/vnd.android.package-archive,binary/octet-stream,application/octet-stream,application/pdf,application/x-font-ttf,application/x-font-otf,application/x-font-woff,application/vnd.ms-fontobject,image/avif
+contentExclude: text/css,image/jpeg,image/jpg,image/png,image/svg+xml,image/gif,image/tiff,image/webp,image/bmp,image/x-icon,image/vnd.microsoft.icon,font/ttf,font/woff,font/woff2,font/x-woff2,font/x-woff,font/otf,audio/mpeg,audio/wav,audio/webm,audio/aac,audio/ogg,audio/wav,audio/webm,video/mp4,video/mpeg,video/webm,video/ogg,video/mp2t,video/webm,video/x-msvideo,application/font-woff,application/font-woff2,application/vnd.android.package-archive,binary/octet-stream,application/octet-stream,application/pdf,application/x-font-ttf,application/x-font-otf,application/x-font-woff,application/vnd.ms-fontobject,image/avif,application/zip,application/x-zip-compressed,application/x-msdownload,application/x-apple-diskimage,application/x-rpm,application/vnd.debian.binary-package
 fileExtExclude: .zip,.dmg,.rpm,.deb,.gz,.tar,.jpg,.jpeg,.png,.svg,.img,.gif,.mp4,.flv,.ogv,.webm,.webp,.mov,.mp3,.m4a,.m4p,.scss,.tif,.tiff,.ttf,.otf,.woff,.woff2,.bmp,.ico,.eot,.htc,.rtf,.swf,.image,.wav,.gltf,.pict,.svgz,.eps,.midi,.mid
 regexFiles: php|php3|php5|asp|aspx|ashx|cfm|cgi|pl|jsp|jspx|json|js|action|html|xhtml|htm|bak|do|txt|wsdl|wadl|xml|xls|xlsx|bin|conf|config|bz2|bzip2|gzip|tar\.gz|tgz|log|src|zip|js\.map
 respParamLinksFound: True

xnLinkFinder.py

@@ -21,7 +21,7 @@
 lstStopWords = {}
 lstPathWords = set() 
 extraStopWords = ""
-lstWordsContentTypes = {}
+contentTypesProcessed = set()
 lstExclusions = {}
 lstFileExtExclusions = {}
 requestHeaders = {}
@@ -124,7 +124,7 @@ class StopProgram(enum.Enum):
 
 # A comma separated list of Content-Type exclusions used when the exclusions from config.yml cannot be found
 # These content types will NOT be checked
-DEFAULT_CONTENTTYPE_EXCLUSIONS = "text/css,image/jpeg,image/jpg,image/png,image/svg+xml,image/gif,image/tiff,image/webp,image/bmp,image/x-icon,image/vnd.microsoft.icon,font/ttf,font/woff,font/woff2,font/x-woff2,font/x-woff,font/otf,audio/mpeg,audio/wav,audio/webm,audio/aac,audio/ogg,audio/wav,audio/webm,video/mp4,video/mpeg,video/webm,video/ogg,video/mp2t,video/webm,video/x-msvideo,application/font-woff,application/font-woff2,application/vnd.android.package-archive,binary/octet-stream,application/octet-stream,application/pdf,application/x-font-ttf,application/x-font-otf,application/x-font-woff,application/vnd.ms-fontobject,image/avif"
+DEFAULT_CONTENTTYPE_EXCLUSIONS = "text/css,image/jpeg,image/jpg,image/png,image/svg+xml,image/gif,image/tiff,image/webp,image/bmp,image/x-icon,image/vnd.microsoft.icon,font/ttf,font/woff,font/woff2,font/x-woff2,font/x-woff,font/otf,audio/mpeg,audio/wav,audio/webm,audio/aac,audio/ogg,audio/wav,audio/webm,video/mp4,video/mpeg,video/webm,video/ogg,video/mp2t,video/webm,video/x-msvideo,application/font-woff,application/font-woff2,application/vnd.android.package-archive,binary/octet-stream,application/octet-stream,application/pdf,application/x-font-ttf,application/x-font-otf,application/x-font-woff,application/vnd.ms-fontobject,image/avif,application/zip,application/x-zip-compressed,application/x-msdownload,application/x-apple-diskimage,application/x-rpm,application/vnd.debian.binary-package"
 
 # A comma separated list of file extension exclusions used when the file ext exclusions from config.yml cannot be found
 # In Directory mode, files with these extensions will NOT be checked
@@ -365,24 +365,27 @@ def includeLink(link):
     return include
 
 
-def includeFile(fileName):
+def includeFile(fileOrUrl):
     """
-    Determine if the passed file name should be excluded by checking the list of exclusions
+    Determine if the passed file name (or URL) should be excluded by checking the list of exclusions
     Returns whether the file should be included
     """
     try:
         global lstFileExtExclusions
 
         include = True
 
+        # If a URL is passed, we want to remove any query string or fragment
+        fileOrUrl = fileOrUrl.split("?")[0].split("#")[0].lower()
+
         # Go through lstFileExtExclusions and see if finding contains any. If not then continue
         for exc in lstFileExtExclusions:
             try:
-                if fileName.endswith(exc):
+                if fileOrUrl.endswith(exc.lower()):
                     include = False
             except Exception as e:
                 if vverbose():
-                    writerr(colored("ERROR includeFile 2: Failed to check exclusions for a finding on file: " + fileName + " (" + str(e) + ")", "red"))
+                    writerr(colored("ERROR includeFile 2: Failed to check exclusions for a finding on file/url: " + fileOrUrl + " (" + str(e) + ")", "red"))
 
     except Exception as e:
         if vverbose():
@@ -391,7 +394,7 @@ def includeFile(fileName):
     return include
 
 
-def includeContentType(header):
+def includeContentType(header,url):
     """
     Determine if the content type is in the exclusions
     Returns whether the content type is included
@@ -420,10 +423,25 @@ def includeContentType(header):
         for excludeContentType in lstExcludeContentType:
             if contentType.lower() == excludeContentType.lower():
                 include = False
+
+        # If the content type can be included and -vv option was passed, add to the set to display at the end
+        if vverbose():
+            try:
+                if include and contentType != '':
+                    contentTypesProcessed.add(contentType)
+            except:
+                pass
+
+        # If the content type wasn't found, check against file extensions
+        if contentType == "":
+            url = url.split("?")[0].split("#")[0].split("/")[-1]
+            if url.find(".") > 0:
+                include = includeFile(url)
+
     except Exception as e:
         if vverbose():
             writerr(colored("ERROR includeContentType 1: " + str(e), "red"))
-
+            
     return include
 
 
@@ -534,7 +552,7 @@ def getResponseLinks(response, url):
         try:
             # If it is content-type we want to process then carry on, or if a directory was passed (so there is no content type) ensure the filename is not an exclusion
             if (dirPassed and includeFile(url)) or (
-                not dirPassed and includeContentType(header)
+                not dirPassed and includeContentType(header,responseUrl)
             ):
                 reString = (
                     r"(?:^|\"|'|\\n|\\r|\n|\r|\s?)(((?:[a-zA-Z]{1,10}:\/\/|\/\/)([^\"'\/]{1,}\.[a-zA-Z]{2,}|localhost)[^\"'\n\s]{0,})|((?:\/|\.\.\/|\.\/)[^\"'><,;| *()(%%$^\/\\\[\]][^\"'><,;|()\s]{1,})|([a-zA-Z0-9_\-\/]{1,}\/[a-zA-Z0-9_\-\/]{1,}\.(?:[a-zA-Z]{1,4}"
@@ -1409,6 +1427,10 @@ def processOutput():
     """
 
     try:
+        # Show the content types processed if verbose mode is on
+        if vverbose() and len(contentTypesProcessed) > 0:
+            write(colored("\nContent-types processed: ","cyan")+colored(str(contentTypesProcessed)+"\n","white"))
+
         # Process output of the found links
         processLinkOutput()
 
@@ -2624,7 +2646,7 @@ def processEachInput(input):
     """
     Process the input, whether its from -i or <stdin>
     """
-    global burpFile, zapFile, urlPassed, stdFile, stdinFile, dirPassed, stdinMultiple, linksFound, linksVisited, totalRequests, skippedRequests, failedRequests, paramsFound, waymoreMode, stopProgram
+    global burpFile, zapFile, urlPassed, stdFile, stdinFile, dirPassed, stdinMultiple, linksFound, linksVisited, totalRequests, skippedRequests, failedRequests, paramsFound, waymoreMode, stopProgram, contentTypesProcessed
 
     if stopProgram is None:
         checkMaxTimeLimit()
@@ -2764,6 +2786,7 @@ def processEachInput(input):
             linksFound = set()
             linksVisited = set()
             paramsFound = set()
+            contentTypesProcessed = set()
             totalRequests = 0
             skippedRequests = 0
             failedRequests = 0
@@ -3673,6 +3696,7 @@ def checkMaxTimeLimit():
             linksFound = set()
             linksVisited = set()
             paramsFound = set()
+            contentTypesProcessed = set()
             totalRequests = 0
             skippedRequests = 0
             failedRequests = 0

xnLinkFinder/__init__.py

@@ -1 +1 @@
-__version__="3.7"
+__version__="3.8"