Docker updates, fixes (smicallef#1386)

* dockerfile fixes

* updated for new data directory

* dockerfile fixes

* updated for new data directory

* change data directory to match $SPIDERFOOT_DATA, create if missing

* consolidate data, logs, and cache directories, fix dockerfile indentation

Dockerfile

@@ -51,10 +51,11 @@ RUN pip3 install -r "$REQUIREMENTS"
 
 FROM alpine:3.13.0
 WORKDIR /home/spiderfoot
-ENV SPIDERFOOT_LOGS /home/spiderfoot/log
 
-# Place database and configs outside installation directory
+# Place database and logs outside installation directory
 ENV SPIDERFOOT_DATA /var/lib/spiderfoot
+ENV SPIDERFOOT_LOGS /var/lib/spiderfoot/log
+ENV SPIDERFOOT_CACHE /var/lib/spiderfoot/cache
 
 # Run everything as one command so that only one layer is created
 RUN apk --update --no-cache add python3 musl openssl libxslt tinyxml libxml2 jpeg zlib openjpeg \
@@ -64,10 +65,12 @@ RUN apk --update --no-cache add python3 musl openssl libxslt tinyxml libxml2 jpe
     && rm -rf /var/cache/apk/* \
     && rm -rf /lib/apk/db \
     && rm -rf /root/.cache \
-    && mkdir $SPIDERFOOT_DATA \
-    && mkdir $SPIDERFOOT_LOGS \
+    && mkdir -p $SPIDERFOOT_DATA || true \
+    && mkdir -p $SPIDERFOOT_LOGS || true \
+    && mkdir -p $SPIDERFOOT_CACHE || true \
+    && chown spiderfoot:spiderfoot $SPIDERFOOT_DATA \
     && chown spiderfoot:spiderfoot $SPIDERFOOT_LOGS \
-    && chown spiderfoot:spiderfoot $SPIDERFOOT_DATA
+    && chown spiderfoot:spiderfoot $SPIDERFOOT_CACHE
 
 COPY . .
 COPY --from=build /opt/venv /opt/venv

Dockerfile.full

@@ -16,7 +16,7 @@ RUN apt-get -y update && apt-get -y install nmap
 RUN mkdir /tools || true
 WORKDIR /tools
 RUN git clone --depth=1 https://github.com/blechschmidt/massdns.git \
-   && cd massdns && make && make install && cd /tools && rm -r massdns
+    && cd massdns && make && make install && cd /tools && rm -r massdns
 
 # Install Golang tools
 RUN apt-get -y update && apt-get -y install golang
@@ -30,54 +30,60 @@ RUN GO111MODULE=on go get -v github.com/projectdiscovery/shuffledns/cmd/shuffled
 RUN apt-get -y update && apt-get -y install ruby ruby-dev bundler
 # WhatWeb
 RUN git clone https://github.com/urbanadventurer/WhatWeb \
-   && gem install rchardet mongo json && cd /tools/WhatWeb \
-   && bundle install && cd /tools
+    && gem install rchardet mongo json && cd /tools/WhatWeb \
+    && bundle install && cd /tools
 
 RUN groupadd spiderfoot \
     && useradd -m -g spiderfoot -d /home/spiderfoot -s /sbin/nologin \
     -c "SpiderFoot User" spiderfoot
 
-ENV SPIDERFOOT_LOGS /home/spiderfoot/log
+# Place database and logs outside installation directory
 ENV SPIDERFOOT_DATA /var/lib/spiderfoot
-RUN mkdir -p  "$SPIDERFOOT_LOGS" "$SPIDERFOOT_DATA" \
-   && chown spiderfoot:spiderfoot "$SPIDERFOOT_LOGS" "$SPIDERFOOT_DATA"
+ENV SPIDERFOOT_LOGS /var/lib/spiderfoot/log
+ENV SPIDERFOOT_CACHE /var/lib/spiderfoot/cache
+
+RUN mkdir -p $SPIDERFOOT_DATA || true \
+    && mkdir -p $SPIDERFOOT_LOGS || true \
+    && mkdir -p $SPIDERFOOT_CACHE || true \
+    && chown spiderfoot:spiderfoot $SPIDERFOOT_DATA \
+    && chown spiderfoot:spiderfoot $SPIDERFOOT_LOGS \
+    && chown spiderfoot:spiderfoot $SPIDERFOOT_CACHE
 
 WORKDIR /home/spiderfoot
 COPY . .
 
 ENV VIRTUAL_ENV=/opt/venv
 RUN mkdir -p "$VIRTUAL_ENV" || true
-RUN chown spiderfoot:spiderfoot "$VIRTUAL_ENV"
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN python -m venv "$VIRTUAL_ENV"
+
+ARG REQUIREMENTS=requirements.txt
+COPY "$REQUIREMENTS" requirements.txt
 
 RUN chown -R spiderfoot:spiderfoot /tools
-WORKDIR /tools
+RUN chown -R spiderfoot:spiderfoot "$VIRTUAL_ENV"
+RUN chown -R spiderfoot:spiderfoot "/home/spiderfoot"
 
 USER spiderfoot
 
-ARG REQUIREMENTS=requirements.txt
-COPY "$REQUIREMENTS" requirements.txt
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-RUN python -m venv "$VIRTUAL_ENV"
 RUN pip install -U pip
 RUN pip install -r "$REQUIREMENTS"
 
 # Install Python tools
 RUN pip install dnstwist
 # CMSeeK
+WORKDIR /tools
 RUN git clone https://github.com/Tuhinshubhra/CMSeeK && cd CMSeeK \
-   && pip install -r requirements.txt && mkdir Results && cd /tools
+    && pip install -r requirements.txt && mkdir Results
 
 WORKDIR /home/spiderfoot
 
 EXPOSE 5001
 
-# Configure tool paths
-ENTRYPOINT python -c 'from spiderfoot import SpiderFootDb; \
-db = SpiderFootDb({"__database": "spiderfoot.db"}, init=True); \
+# Run the application
+CMD python -c 'from spiderfoot import SpiderFootDb; \
+db = SpiderFootDb({"__database": "/var/lib/spiderfoot/spiderfoot.db"}); \
 db.configSet({ \
     "sfp_tool_cmseek:cmseekpath": "/tools/CMSeeK/cmseek.py", \
     "sfp_tool_whatweb:whatweb_path": "/tools/WhatWeb/whatweb" \
-})' && /bin/bash
-
-# Run the application.
-CMD ["./sf.py", "-l", "0.0.0.0:5001"]
+})' || true && ./sf.py -l 0.0.0.0:5001

docker-compose.yml

@@ -8,15 +8,25 @@ version: "3"
 #
 # Full image (all CLI tools installed):
 #     $ docker-compose -f docker-compose.yml -f docker-compose-full.yml up
+#
+# Spiderfoot data resides in a Docker volume
+#
+#     $ ls -lh /var/lib/docker/volumes/spiderfoot_spiderfoot-data/_data
+#       total 104K
+#       drwxr-xr-x 2 user user 4.0K Sep 22 09:51 cache
+#       -rw-r--r-- 1 user user 100K Sep 22 15:19 spiderfoot.db
 
 services:
   spiderfoot:
     build:
-       context: ./
+      context: ./
     volumes:
-      - ./spiderfoot.db:/home/spiderfoot/spiderfoot.db
+      - spiderfoot-data:/var/lib/spiderfoot
     image: spiderfoot
     container_name: spiderfoot
     ports:
       - "5001:5001"
     restart: unless-stopped
+
+volumes:
+  spiderfoot-data:

spiderfoot/db.py

@@ -10,6 +10,7 @@
 # Licence:     GPL
 # -------------------------------------------------------------------------------
 
+from pathlib import Path
 import re
 import sqlite3
 import threading
@@ -276,6 +277,9 @@ def __init__(self, opts, init=False):
 
         database_path = opts['__database']
 
+        # create database directory
+        Path(database_path).parent.mkdir(exist_ok=True, parents=True)
+
         # connect() will create the database file if it doesn't exist, but
         # at least we can use this opportunity to ensure we have permissions to
         # read and write to such a file.