"Dockerfile.full" containing all CLI tools (smicallef#1322)

* Dockerfile.full containing all CLI tools

Dockerfile.full

@@ -0,0 +1,83 @@
+#
+# Spiderfoot Dockerfile (Full - includes all CLI tools, etc.)
+#
+# http://www.spiderfoot.net
+#
+# Written by: TheTechromancer
+#
+
+FROM python:3
+
+# Install tools/dependencies from apt
+RUN apt-get -y update && apt-get -y install nmap
+
+# Compile other tools from source
+# MassDNS
+RUN mkdir /tools || true
+WORKDIR /tools
+RUN git clone --depth=1 https://github.com/blechschmidt/massdns.git \
+   && cd massdns && make && make install && cd /tools && rm -r massdns
+
+# Install Golang tools
+RUN apt-get -y update && apt-get -y install golang
+ENV GOPATH="/go"
+ENV PATH="$GOPATH/bin:$PATH"
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin"
+# ShuffleDNS
+RUN GO111MODULE=on go get -v github.com/projectdiscovery/shuffledns/cmd/shuffledns
+
+# Install Ruby tools
+RUN apt-get -y update && apt-get -y install ruby ruby-dev bundler
+# WhatWeb
+RUN git clone https://github.com/urbanadventurer/WhatWeb \
+   && gem install rchardet mongo json && cd /tools/WhatWeb \
+   && bundle install && cd /tools
+
+RUN groupadd spiderfoot \
+    && useradd -m -g spiderfoot -d /home/spiderfoot -s /sbin/nologin \
+    -c "SpiderFoot User" spiderfoot
+
+ENV SPIDERFOOT_LOGS /home/spiderfoot/log
+ENV SPIDERFOOT_DATA /var/lib/spiderfoot
+RUN mkdir -p  "$SPIDERFOOT_LOGS" "$SPIDERFOOT_DATA" \
+   && chown spiderfoot:spiderfoot "$SPIDERFOOT_LOGS" "$SPIDERFOOT_DATA"
+
+WORKDIR /home/spiderfoot
+COPY . .
+
+ENV VIRTUAL_ENV=/opt/venv
+RUN mkdir -p "$VIRTUAL_ENV" || true
+RUN chown spiderfoot:spiderfoot "$VIRTUAL_ENV"
+
+RUN chown -R spiderfoot:spiderfoot /tools
+WORKDIR /tools
+
+USER spiderfoot
+
+ARG REQUIREMENTS=requirements.txt
+COPY "$REQUIREMENTS" requirements.txt
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN python -m venv "$VIRTUAL_ENV"
+RUN pip install -U pip
+RUN pip install -r "$REQUIREMENTS"
+
+# Install Python tools
+RUN pip install dnstwist
+# CMSeeK
+RUN git clone https://github.com/Tuhinshubhra/CMSeeK && cd CMSeeK \
+   && pip install -r requirements.txt && mkdir Results && cd /tools
+
+WORKDIR /home/spiderfoot
+
+EXPOSE 5001
+
+# Configure tool paths
+ENTRYPOINT python -c 'from spiderfoot import SpiderFootDb; \
+db = SpiderFootDb({"__database": "spiderfoot.db"}, init=True); \
+db.configSet({ \
+    "sfp_tool_cmseek:cmseekpath": "/tools/CMSeeK/cmseek.py", \
+    "sfp_tool_whatweb:whatweb_path": "/tools/WhatWeb/whatweb" \
+})' && /bin/bash
+
+# Run the application.
+CMD ["./sf.py", "-l", "0.0.0.0:5001"]

docker-compose-dev.yml

@@ -0,0 +1,6 @@
+version: "3"
+
+services:
+  spiderfoot:
+    volumes:
+      - .:/home/spiderfoot

docker-compose-full.yml

@@ -0,0 +1,7 @@
+version: "3"
+
+services:
+  spiderfoot:
+    build:
+      context: ./
+      dockerfile: ./Dockerfile.full

docker-compose.yml

@@ -1,9 +1,20 @@
 version: "3"
 
+# Basic usage:
+#     $ docker-compose up
+#
+# Dev environment (code directory mapped into container):
+#     $ docker-compose -f docker-compose.yml -f docker-compose-dev.yml up
+#
+# Full image (all CLI tools installed):
+#     $ docker-compose -f docker-compose.yml -f docker-compose-full.yml up
+
 services:
   spiderfoot:
     build:
        context: ./
+    volumes:
+      - ./spiderfoot.db:/home/spiderfoot/spiderfoot.db
     image: spiderfoot
     container_name: spiderfoot
     ports:

modules/sfp_tool_dnstwist.py

@@ -12,7 +12,8 @@
 # -------------------------------------------------------------------------------
 
 import json
-import os.path
+from pathlib import Path
+from shutil import which
 from subprocess import PIPE, Popen
 
 from spiderfoot import SpiderFootEvent, SpiderFootPlugin
@@ -46,7 +47,7 @@ class sfp_tool_dnstwist(SpiderFootPlugin):
     # Option descriptions
     optdescs = {
         'pythonpath': "Path to Python interpreter to use for DNSTwist. If just 'python' then it must be in your PATH.",
-        'dnstwistpath': "Path to the where the dnstwist.py file lives. Must be set."
+        'dnstwistpath': "Path to the where the dnstwist.py file lives. Optional."
     }
 
     results = None
@@ -88,32 +89,38 @@ def handleEvent(self, event):
 
         self.results[eventData] = True
 
-        if not self.opts['dnstwistpath']:
-            self.sf.error("You enabled sfp_tool_dnstwist but did not set a path to the tool!")
-            self.errorState = True
-            return
-
-        # Normalize path
-        if self.opts['dnstwistpath'].endswith('dnstwist.py'):
-            exe = self.opts['dnstwistpath']
-        elif self.opts['dnstwistpath'].endswith('/'):
-            exe = self.opts['dnstwistpath'] + "dnstwist.py"
+        dnstwistLocation = which('dnstwist')
+        if dnstwistLocation and Path(dnstwistLocation).is_file():
+            cmd = ['dnstwist']
         else:
-            exe = self.opts['dnstwistpath'] + "/dnstwist.py"
+            if not self.opts['dnstwistpath']:
+                self.sf.error("You enabled sfp_tool_dnstwist but did not set a path to the tool!")
+                self.errorState = True
+                return
 
-        # If tool is not found, abort
-        if not os.path.isfile(exe):
-            self.sf.error("File does not exist: " + exe)
-            self.errorState = True
-            return
+            # Normalize path
+            if self.opts['dnstwistpath'].endswith('dnstwist.py'):
+                exe = self.opts['dnstwistpath']
+            elif self.opts['dnstwistpath'].endswith('/'):
+                exe = self.opts['dnstwistpath'] + "dnstwist.py"
+            else:
+                exe = self.opts['dnstwistpath'] + "/dnstwist.py"
+
+            # If tool is not found, abort
+            if not Path(exe).is_file():
+                self.sf.error("File does not exist: " + exe)
+                self.errorState = True
+                return
+
+            cmd = [self.opts['pythonpath'], exe]
 
         # Sanitize domain name.
         if not self.sf.sanitiseInput(eventData):
             self.sf.error("Invalid input, refusing to run.")
             return
 
         try:
-            p = Popen([self.opts['pythonpath'], exe, "-f", "json", "-r", eventData], stdout=PIPE, stderr=PIPE)
+            p = Popen(cmd + ["-f", "json", "-r", eventData], stdout=PIPE, stderr=PIPE)
             stdout, stderr = p.communicate(input=None)
             if p.returncode == 0:
                 content = stdout