feat: email ingestion API (#7342)

* feat: IANA review email ingestor API

* refactor: Replace iana email api with generic one

* chore: Add type hint

* feat: Ingest ipr responses

* feat: Ingest nomcom feedback

* refactor: message -> msg

* fix: Typo

* feat: Send email on nomcom ingestion failure

* feat: Send email on IPR mail ingestion error

* feat: Check content type, handle more errs

* fix: drop additionalProperties: false

Unfortunately this does not mix well with
the conditional "year" property.

* test: Test ingest_email view

* Revert "test: Test ingest_email view"

This reverts commit e498022.

* test: Test ingest_email view

* fix: pass new test

* test: Test ingest_review_email

* fix: Pass new test

* test: Test ipr ingest_response_email

* fix: pass new test

* test: test nomcom ingest_feedback_email

* chore: fix typo found in code reviw

* fix: De-lint

ietf/api/tests.py

@@ -1,6 +1,6 @@
 # Copyright The IETF Trust 2015-2020, All Rights Reserved
 # -*- coding: utf-8 -*-
-
+import base64
 import datetime
 import json
 import html
@@ -36,11 +36,12 @@
 from ietf.person.models import Email, User
 from ietf.person.models import PersonalApiKey
 from ietf.stats.models import MeetingRegistration
-from ietf.utils.mail import outbox, get_payload_text
+from ietf.utils.mail import empty_outbox, outbox, get_payload_text
 from ietf.utils.models import DumpInfo
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized, reload_db_objects
 
 from .ietf_utils import is_valid_token, requires_api_token
+from .views import EmailIngestionError
 
 OMITTED_APPS = (
     'ietf.secr.meetings',
@@ -1013,6 +1014,194 @@ def test_role_holder_addresses(self):
             sorted(e.address for e in emails),
         )
 
+    @override_settings(APP_API_TOKENS={"ietf.api.views.ingest_email": "valid-token"})
+    @mock.patch("ietf.api.views.iana_ingest_review_email")
+    @mock.patch("ietf.api.views.ipr_ingest_response_email")
+    @mock.patch("ietf.api.views.nomcom_ingest_feedback_email")
+    def test_ingest_email(
+        self, mock_nomcom_ingest, mock_ipr_ingest, mock_iana_ingest
+    ):
+        mocks = {mock_nomcom_ingest, mock_ipr_ingest, mock_iana_ingest}
+        empty_outbox()        
+        url = urlreverse("ietf.api.views.ingest_email")
+
+        # test various bad calls
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 403)
+        self.assertFalse(any(m.called for m in mocks))
+
+        r = self.client.post(url)
+        self.assertEqual(r.status_code, 403)
+        self.assertFalse(any(m.called for m in mocks))
+
+        r = self.client.get(url, headers={"X-Api-Key": "valid-token"})
+        self.assertEqual(r.status_code, 405)
+        self.assertFalse(any(m.called for m in mocks))
+
+        r = self.client.post(url, headers={"X-Api-Key": "valid-token"})
+        self.assertEqual(r.status_code, 415)
+        self.assertFalse(any(m.called for m in mocks))
+
+        r = self.client.post(
+            url, content_type="application/json", headers={"X-Api-Key": "valid-token"}
+        )
+        self.assertEqual(r.status_code, 400)
+        self.assertFalse(any(m.called for m in mocks))
+
+        r = self.client.post(
+            url,
+            "this is not JSON!",
+            content_type="application/json",
+            headers={"X-Api-Key": "valid-token"},
+        )
+        self.assertEqual(r.status_code, 400)
+        self.assertFalse(any(m.called for m in mocks))
+
+        r = self.client.post(
+            url,
+            {"json": "yes", "valid_schema": False},
+            content_type="application/json",
+            headers={"X-Api-Key": "valid-token"},
+        )
+        self.assertEqual(r.status_code, 400)
+        self.assertFalse(any(m.called for m in mocks))
+
+        # test that valid requests call handlers appropriately
+        message_b64 = base64.b64encode(b"This is a message").decode()
+        r = self.client.post(
+            url,
+            {"dest": "iana-review", "message": message_b64},
+            content_type="application/json",
+            headers={"X-Api-Key": "valid-token"},
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertTrue(mock_iana_ingest.called)
+        self.assertEqual(mock_iana_ingest.call_args, mock.call(b"This is a message"))
+        self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))
+        mock_iana_ingest.reset_mock()
+
+        r = self.client.post(
+            url,
+            {"dest": "ipr-response", "message": message_b64},
+            content_type="application/json",
+            headers={"X-Api-Key": "valid-token"},
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertTrue(mock_ipr_ingest.called)
+        self.assertEqual(mock_ipr_ingest.call_args, mock.call(b"This is a message"))
+        self.assertFalse(any(m.called for m in (mocks - {mock_ipr_ingest})))
+        mock_ipr_ingest.reset_mock()
+
+        r = self.client.post(
+            url,
+            {"dest": "nomcom-feedback", "message": message_b64, "year": 2024},  # arbitrary year
+            content_type="application/json",
+            headers={"X-Api-Key": "valid-token"},
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertTrue(mock_nomcom_ingest.called)
+        self.assertEqual(mock_nomcom_ingest.call_args, mock.call(b"This is a message", 2024))
+        self.assertFalse(any(m.called for m in (mocks - {mock_nomcom_ingest})))
+        mock_nomcom_ingest.reset_mock()
+
+        # test that exceptions lead to email being sent - assumes that iana-review handling is representative
+        mock_iana_ingest.side_effect = EmailIngestionError("Error: don't send email")
+        r = self.client.post(
+            url,
+            {"dest": "iana-review", "message": message_b64},
+            content_type="application/json",
+            headers={"X-Api-Key": "valid-token"},
+        )
+        self.assertEqual(r.status_code, 400)
+        self.assertTrue(mock_iana_ingest.called)
+        self.assertEqual(mock_iana_ingest.call_args, mock.call(b"This is a message"))
+        self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))
+        self.assertEqual(len(outbox), 0)  # implicitly tests that _none_ of the earlier tests sent email
+        mock_iana_ingest.reset_mock()
+
+        # test default recipients and attached original message
+        mock_iana_ingest.side_effect = EmailIngestionError(
+            "Error: do send email",
+            email_body="This is my email\n",
+            email_original_message=b"This is the original message"
+        )
+        with override_settings(ADMINS=[("Some Admin", "admin@example.com")]):
+            r = self.client.post(
+                url,
+                {"dest": "iana-review", "message": message_b64},
+                content_type="application/json",
+                headers={"X-Api-Key": "valid-token"},
+            )
+        self.assertEqual(r.status_code, 400)
+        self.assertTrue(mock_iana_ingest.called)
+        self.assertEqual(mock_iana_ingest.call_args, mock.call(b"This is a message"))
+        self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))
+        self.assertEqual(len(outbox), 1)
+        self.assertIn("admin@example.com", outbox[0]["To"])
+        self.assertEqual("Error: do send email", outbox[0]["Subject"])
+        self.assertEqual("This is my email\n", get_payload_text(outbox[0].get_body()))
+        attachments = list(a for a in outbox[0].iter_attachments())
+        self.assertEqual(len(attachments), 1)
+        self.assertEqual(attachments[0].get_filename(), "original-message")
+        self.assertEqual(attachments[0].get_content_type(), "application/octet-stream")
+        self.assertEqual(attachments[0].get_content(), b"This is the original message")
+        mock_iana_ingest.reset_mock()
+        empty_outbox()
+
+        # test overridden recipients and no attached original message
+        mock_iana_ingest.side_effect = EmailIngestionError(
+            "Error: do send email",
+            email_body="This is my email\n",
+            email_recipients=("thatguy@example.com")
+        )
+        with override_settings(ADMINS=[("Some Admin", "admin@example.com")]):
+            r = self.client.post(
+                url,
+                {"dest": "iana-review", "message": message_b64},
+                content_type="application/json",
+                headers={"X-Api-Key": "valid-token"},
+            )
+        self.assertEqual(r.status_code, 400)
+        self.assertTrue(mock_iana_ingest.called)
+        self.assertEqual(mock_iana_ingest.call_args, mock.call(b"This is a message"))
+        self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))
+        self.assertEqual(len(outbox), 1)
+        self.assertNotIn("admin@example.com", outbox[0]["To"])
+        self.assertIn("thatguy@example.com", outbox[0]["To"])
+        self.assertEqual("Error: do send email", outbox[0]["Subject"])
+        self.assertEqual("This is my email\n", get_payload_text(outbox[0]))
+        mock_iana_ingest.reset_mock()
+        empty_outbox()
+
+        # test attached traceback
+        mock_iana_ingest.side_effect = EmailIngestionError(
+            "Error: do send email",
+            email_body="This is my email\n",
+            email_attach_traceback=True,
+        )
+        with override_settings(ADMINS=[("Some Admin", "admin@example.com")]):
+            r = self.client.post(
+                url,
+                {"dest": "iana-review", "message": message_b64},
+                content_type="application/json",
+                headers={"X-Api-Key": "valid-token"},
+            )
+        self.assertEqual(r.status_code, 400)
+        self.assertTrue(mock_iana_ingest.called)
+        self.assertEqual(mock_iana_ingest.call_args, mock.call(b"This is a message"))
+        self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))
+        self.assertEqual(len(outbox), 1)
+        self.assertIn("admin@example.com", outbox[0]["To"])
+        self.assertEqual("Error: do send email", outbox[0]["Subject"])
+        self.assertEqual("This is my email\n", get_payload_text(outbox[0].get_body()))
+        attachments = list(a for a in outbox[0].iter_attachments())
+        self.assertEqual(len(attachments), 1)
+        self.assertEqual(attachments[0].get_filename(), "traceback.txt")
+        self.assertEqual(attachments[0].get_content_type(), "text/plain")
+        self.assertIn("ietf.api.views.EmailIngestionError: Error: do send email", attachments[0].get_content())
+        mock_iana_ingest.reset_mock()
+        empty_outbox()
+
 
 class DirectAuthApiTests(TestCase):
 

ietf/api/urls.py

@@ -24,7 +24,9 @@
     # --- Custom API endpoints, sorted alphabetically ---
     # Email alias information for drafts
     url(r'^doc/draft-aliases/$', api_views.draft_aliases),
-    # GPRD: export of personal information for the logged-in person
+    # email ingestor
+    url(r'email/$', api_views.ingest_email),
+    # GDPR: export of personal information for the logged-in person
     url(r'^export/personal-information/$', api_views.PersonalInformationExportView.as_view()),
     # Email alias information for groups
     url(r'^group/group-aliases/$', api_views.group_aliases),