feat(s3): add s3 ssec option

.mypy.ini

@@ -51,3 +51,9 @@ ignore_missing_imports = True
 
 [mypy-django_filters.*]
 ignore_missing_imports = True
+
+[mypy-tqdm.*]
+ignore_missing_imports = True
+
+[mypy-storages.*]
+ignore_missing_imports = True

CONFIGURATION.md

@@ -117,9 +117,9 @@ If either `EMAIL_HOST_USER` or `EMAIL_HOST_PASSWORD` is empty, Django won't atte
 - `MEDIA_ROOT`: Absolute filesystem path to the directory that will hold user-uploaded files. (default: "")
 - `MEDIA_URL`: URL that handles the media served from MEDIA_ROOT, used for managing stored files. When using buckets this needs to be changed. (default: `api/v1/template/`)
 
-### [django-storages](https://django-storages.readthedocs.io/en/1.13.2/backends/amazon-S3.html) S3 settings
+### [django-storages](https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html) S3 settings
 
-Refer to for example [Digital Ocean](https://django-storages.readthedocs.io/en/1.13.2/backends/digital-ocean-spaces.html) configuration if using a S3 compatible storage which isn't AWS.
+Refer to for example [Digital Ocean](https://django-storages.readthedocs.io/en/latest/backends/s3_compatible/digital-ocean-spaces.html) configuration if using a S3 compatible storage which isn't AWS.
 
 Required to use S3 storage:
 
@@ -136,3 +136,5 @@ Optional:
 - `AWS_S3_SIGNATURE_VERSION`: S3 signature version to use (default: `s2`)
 - `AWS_S3_USE_SSL`: Whether or not to use SSL when connecting to S3 (default: `True`)
 - `AWS_S3_VERIFY`: Whether or not to verify the connection to S3. Can be set to False to not verify SSL/TLS certificates. (default: `None`)
+- `DMS_ENABLE_AT_REST_ENCRYPTION`: Whether to use SSEC to encrypt files uploaded to S3 (default: `False`)
+- `DMS_S3_STORAGE_SSEC_SECRET`: Secret key for SSEC encryption, has to be 32 bytes long (default: `xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`)

document_merge_service/api/management/commands/dms_encrypt_templates.py

@@ -0,0 +1,57 @@
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+from django.core.files.storage import storages
+from django.core.management.base import BaseCommand
+from tqdm import tqdm
+
+from document_merge_service.api.models import Template
+
+
+class Command(BaseCommand):
+    help = "Swaps plain text template content to encrypted content"
+
+    def handle(self, *args, **options):
+        if not settings.DMS_ENABLE_AT_REST_ENCRYPTION:
+            return self.stdout.write(
+                self.style.WARNING(
+                    "Encryption is not enabled. Skipping encryption of templates."
+                )
+            )
+
+        failed_templates = []
+
+        # flip between default and encrypted storage to have the correct parameters in the requests
+        encrypted_storage = storages.create_storage(settings.STORAGES["default"])
+        unencrypted_storage_setting = settings.STORAGES["default"]
+        if (
+            "OPTIONS" not in unencrypted_storage_setting
+            or "object_parameters" not in unencrypted_storage_setting["OPTIONS"]
+        ):
+            raise ImproperlyConfigured(
+                "Encryption is enabled but no object_parameters found in the storage settings."
+            )
+        del unencrypted_storage_setting["OPTIONS"]["object_parameters"]
+        unencrypted_storage = storages.create_storage(unencrypted_storage_setting)
+
+        query = Template.objects.all()
+        for template in tqdm(query.iterator(50), total=query.count()):
+            # get original template content
+            template.template.storage = unencrypted_storage
+            try:
+                content = template.template.open()
+
+                # overwrite with encrypted content
+                template.template.storage = encrypted_storage
+                template.template.save(template.template.name, content)
+            except Exception as e:
+                self.stdout.write(
+                    self.style.WARNING(f"Error for template {str(template.pk)}: {e}")
+                )
+                failed_templates.append(str(template.pk))
+                continue
+
+        if failed_templates:
+            self.stdout.write(
+                self.style.WARNING(f"These templates failed:\n{failed_templates}")
+            )
+        self.stdout.write(self.style.SUCCESS("Encryption finished"))

document_merge_service/api/tests/test_encrypt_templates.py

@@ -0,0 +1,92 @@
+from copy import deepcopy
+from io import StringIO
+
+import pytest
+from django.core.exceptions import ImproperlyConfigured
+from django.core.files import File as DjangoFile
+from django.core.management import call_command
+from storages.backends.s3 import S3Storage
+
+from document_merge_service.api.data import django_file
+
+
+@pytest.fixture
+def settings_storage(settings):
+    settings.STORAGES = deepcopy(settings.STORAGES)
+    return settings.STORAGES
+
+
+def test_encrypt_templates(db, settings, settings_storage, mocker, template_factory):
+    template_factory(template=django_file("docx-template.docx"))
+
+    settings.DMS_ENABLE_AT_REST_ENCRYPTION = True
+    settings_storage["default"] = {
+        "BACKEND": "storages.backends.s3.S3Storage",
+        "OPTIONS": {
+            **settings.S3_STORAGE_OPTIONS,
+            "object_parameters": {
+                "SSECustomerKey": "x" * 32,
+                "SSECustomerAlgorithm": "AES256",
+            },
+        },
+    }
+
+    mocker.patch("storages.backends.s3.S3Storage.open")
+    mocker.patch("storages.backends.s3.S3Storage.save")
+    S3Storage.save.return_value = "name-of-the-file"
+    S3Storage.open.return_value = DjangoFile(open("README.md", "rb"))
+
+    call_command("dms_encrypt_templates")
+
+    assert S3Storage.open.call_count == 1
+    assert S3Storage.save.call_count == 1
+
+
+def test_encrypt_templates_disabled(db, template_factory):
+    template_factory(template=django_file("docx-template.docx"))
+
+    out = StringIO()
+    call_command("dms_encrypt_templates", stdout=out)
+
+    assert (
+        "Encryption is not enabled. Skipping encryption of templates." in out.getvalue()
+    )
+
+
+def test_encrypt_template_improperyconfigured(db, settings, template_factory):
+    template_factory(template=django_file("docx-template.docx"))
+    settings.DMS_ENABLE_AT_REST_ENCRYPTION = True
+
+    out = StringIO()
+    with pytest.raises(ImproperlyConfigured):
+        call_command("dms_encrypt_templates", stdout=out)
+
+
+def test_encrypt_templates_failed(
+    db, settings, settings_storage, mocker, template_factory
+):
+    template_factory(template=django_file("docx-template.docx"))
+
+    settings.DMS_ENABLE_AT_REST_ENCRYPTION = True
+    settings_storage["default"] = {
+        "BACKEND": "storages.backends.s3.S3Storage",
+        "OPTIONS": {
+            **settings.S3_STORAGE_OPTIONS,
+            "object_parameters": {
+                "SSECustomerKey": "x" * 32,
+                "SSECustomerAlgorithm": "AES256",
+            },
+        },
+    }
+
+    mocker.patch("storages.backends.s3.S3Storage.open", side_effect=FileNotFoundError)
+    mocker.patch("storages.backends.s3.S3Storage.save")
+    S3Storage.save.return_value = "name-of-the-file"
+    S3Storage.open.return_value = DjangoFile(open("README.md", "rb"))
+
+    out = StringIO()
+    call_command("dms_encrypt_templates", stdout=out)
+
+    assert S3Storage.open.call_count == 1
+    assert S3Storage.save.call_count == 0
+    assert "failed" in out.getvalue()

document_merge_service/settings.py

@@ -163,16 +163,33 @@ def parse_admins(admins):
 MEDIA_URL = env.str("MEDIA_URL", "api/v1/template/")
 
 # django-storages S3 settings
-AWS_S3_ACCESS_KEY_ID = env.str("AWS_S3_ACCESS_KEY_ID", "")
-AWS_S3_SECRET_ACCESS_KEY = env.str("AWS_S3_SECRET_ACCESS_KEY", "")
-AWS_STORAGE_BUCKET_NAME = env.str("AWS_STORAGE_BUCKET_NAME", "")
-AWS_S3_ENDPOINT_URL = env.str("AWS_S3_ENDPOINT_URL", "")
-AWS_S3_REGION_NAME = env.str("AWS_S3_REGION_NAME", "")
-AWS_LOCATION = env.str("AWS_LOCATION", "")
-AWS_S3_FILE_OVERWRITE = env.bool("AWS_S3_FILE_OVERWRITE", False)
-AWS_S3_SIGNATURE_VERSION = env.str("AWS_S3_SIGNATURE_VERSION", "v2")
-AWS_S3_USE_SSL = env.bool("AWS_S3_USE_SSL", default=True)
-AWS_S3_VERIFY = env.bool("AWS_S3_VERIFY", default=None)
+DMS_ENABLE_AT_REST_ENCRYPTION = env.bool("DMS_ENABLE_AT_REST_ENCRYPTION", False)
+S3_STORAGE_OPTIONS = {
+    "access_key": env.str("AWS_S3_ACCESS_KEY_ID", ""),
+    "secret_key": env.str("AWS_S3_SECRET_ACCESS_KEY", ""),
+    "bucket_name": env.str("AWS_STORAGE_BUCKET_NAME", ""),
+    "endpoint_url": env.str("AWS_S3_ENDPOINT_URL", ""),
+    "region_name": env.str("AWS_S3_REGION_NAME", ""),
+    "location": env.str("AWS_LOCATION", ""),
+    "file_overwrite": env.bool("AWS_S3_FILE_OVERWRITE", False),
+    "signature_version": env.str("AWS_S3_SIGNATURE_VERSION", "v2"),
+    "use_ssl": env.bool("AWS_S3_USE_SSL", default=True),
+    "verify": env.bool("AWS_S3_VERIFY", default=None),
+}
+
+if DMS_ENABLE_AT_REST_ENCRYPTION:  # pragma: no cover
+    S3_STORAGE_OPTIONS["object_parameters"] = {
+        "SSECustomerKey": env.str(
+            "DMS_S3_STORAGE_SSEC_SECRET",
+            default=default("x" * 32),
+        ),
+        "SSECustomerAlgorithm": "AES256",
+    }
+
+if (
+    STORAGES["default"]["BACKEND"] == "storages.backends.s3.S3Storage"
+):  # pragma: no cover
+    STORAGES["default"]["OPTIONS"] = S3_STORAGE_OPTIONS
 
 # unoconv
 UNOCONV_ALLOWED_TYPES = env.list("UNOCOV_ALLOWED_TYPES", default=["pdf"])

pyproject.toml

@@ -28,6 +28,7 @@ exclude = [
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<3.13"
+boto3 = "^1.34.143"
 Babel = "^2.15.0"
 Django = "~4.2.13"
 django-cors-headers = "^4.4.0"
@@ -79,6 +80,7 @@ mysql = ["mysqlclient"]
 pgsql = ["psycopg"]
 databases = ["mysqlclient", "psycopg"]
 memcache = ["pymemcache"]
+s3 = ["boto3"]
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]