Fix calculating vacations and commitments for the async process

config/settings/base.py

@@ -518,8 +518,8 @@
 # Group 1. cell's key
 # Group 2. issue number
 SPRINT_ISSUE_REGEX = env.str("SPRINT_ISSUE_REGEX", r"(\w+)-(\d+)")
-# An UTC hour at which the sprint meeting starts.
-SPRINT_MEETING_HOUR_UTC = env.int("SPRINT_MEETING_HOUR_UTC", 16)
+# UTC time at which a new sprint starts (format: `%H:%M`).
+SPRINT_START_TIME_UTC = env.int("SPRINT_MEETING_HOUR_UTC", "00:00")
 # Exact name of the tickets for logging the clean sprint hints.
 SPRINT_MEETINGS_TICKET = env.str("SPRINT_MEETINGS_TICKET", "Meetings")
 
@@ -545,7 +545,8 @@
 GOOGLE_SPILLOVER_SPREADSHEET = env.str("GOOGLE_SPILLOVER_SPREADSHEET")
 GOOGLE_CONTACT_SPREADSHEET = env.str("GOOGLE_CONTACT_SPREADSHEET")
 GOOGLE_AVAILABILITY_RANGE = env.str("GOOGLE_AVAILABILITY_RANGE")
-GOOGLE_AVAILABILITY_REGEX = env.str("GOOGLE_AVAILABILITY_REGEX", r"(\d+).*(pm|am)-(\d+).*(pm|am)")
+# Regex for retrieving users' availability from the "Contact" sheet.
+GOOGLE_AVAILABILITY_REGEX = env.str("GOOGLE_AVAILABILITY_REGEX", r"\d+(?::\d+)?.*?(?:pm|am)")
 GOOGLE_AVAILABILITY_TIME_FORMAT = env.str("GOOGLE_AVAILABILITY_TIME_FORMAT", "%I%p")
 GOOGLE_SPILLOVER_SPREADSHEET_URL = f"https://docs.google.com/spreadsheets/d/{GOOGLE_SPILLOVER_SPREADSHEET}"
 # Spreadsheet with the cell rotations.

sprints/dashboard/libs/google.py

@@ -47,7 +47,7 @@ def get_vacations(from_: str, to: str) -> List[Dict[str, Union[int, str, Dict[st
                 calendarId=calendar,
                 timeZone='Europe/London',
                 timeMin=f'{from_}T00:00:00Z',
-                timeMax=f'{to}T00:00:00Z',
+                timeMax=f'{to}T23:59:59Z',
                 fields='items(end/date, start/date, summary)'
             ).execute()
 

sprints/dashboard/models.py

@@ -2,12 +2,14 @@
 import functools
 import re
 import typing
+from datetime import timedelta
 from typing import (
     Dict,
     List,
     Set,
 )
 
+from dateutil.parser import parse
 from django.conf import settings
 from jira import (
     Issue,
@@ -31,11 +33,11 @@
     get_cell_key,
     get_cell_members,
     get_issue_fields,
+    get_next_sprint,
     get_sprint_end_date,
     get_sprint_meeting_day_division,
     get_sprint_start_date,
     prepare_jql_query,
-    get_next_sprint,
 )
 
 
@@ -252,7 +254,7 @@ def __init__(self, board_id: int, conn: CustomJira) -> None:
         self.cell_key = get_cell_key(conn, board_id)
         self.get_sprints()
         self.create_mock_users()
-        self.vacations = get_vacations(self.future_sprint_start, self.future_sprint_end)
+        self.vacations = get_vacations(self.before_future_sprint_start, self.after_future_sprint_end)
         self.get_issues()
         self.generate_rows()
 
@@ -269,7 +271,15 @@ def get_sprints(self) -> None:
         self.cell_future_sprint = get_next_sprint(sprints['cell'], sprints['cell'][0])
 
         self.future_sprint_start = get_sprint_start_date(self.cell_future_sprint)
-        self.future_sprint_end = get_sprint_end_date(self.cell_future_sprint, sprints['all'])
+        self.future_sprint_end = get_sprint_end_date(self.cell_future_sprint)
+
+        # Helper variables to retrieve more data for different timezones (one extra day on each end of the sprint).
+        self.before_future_sprint_start = (parse(self.future_sprint_start) - timedelta(days=1)).strftime(
+            settings.JIRA_API_DATE_FORMAT
+        )
+        self.after_future_sprint_end = (parse(self.future_sprint_end) + timedelta(days=1)).strftime(
+            settings.JIRA_API_DATE_FORMAT
+        )
 
     def create_mock_users(self):
         """Create mock users for handling unassigned and cross-cell tickets."""
@@ -301,7 +311,7 @@ def get_issues(self) -> None:
         quickfilters: List[QuickFilter] = self.jira_connection.quickfilters(self.board_id)
 
         self.members = get_cell_members(quickfilters)
-        self.sprint_division = get_sprint_meeting_day_division()
+        self.sprint_division = get_sprint_meeting_day_division(self.future_sprint_start)
         self.issues = []
 
         active_sprint_ids = {sprint.id for sprint in self.active_sprints}
@@ -321,8 +331,8 @@ def get_issues(self) -> None:
         for member in self.members:
             schedule = self.jira_connection.user_schedule(
                 member,
-                self.future_sprint_start,
-                self.future_sprint_end,
+                self.before_future_sprint_start,
+                self.after_future_sprint_end,
             )
             self.commitments[member] = {
                 'total': schedule.requiredSeconds,
@@ -377,31 +387,88 @@ def generate_rows(self) -> None:
             if user != self.unassigned_user and user.name not in self.members:
                 self.dashboard.pop(user)
 
-        # Calculate commitments for each user.
+        self._calculate_commitments()
+
+    @typing.no_type_check
+    def _calculate_commitments(self):
+        """
+        Calculates time commitments and vacations for each user.
+        """
         for row in self.rows:
             if row.user != self.unassigned_user:
                 # Calculate vacations
                 for vacation in self.vacations:
-                    if row.user.displayName.startswith(vacation['user']):
+                    if row.user.displayName.startswith(vacation["user"]):
                         for vacation_date in daterange(
-                            max(vacation['start']['date'], self.future_sprint_start),
-                            min(vacation['end']['date'], self.future_sprint_end),
+                            max(
+                                vacation["start"]["date"],
+                                (parse(self.future_sprint_start) - timedelta(days=1)).strftime(
+                                    settings.JIRA_API_DATE_FORMAT
+                                ),
+                            ),
+                            min(
+                                vacation["end"]["date"],
+                                (parse(self.future_sprint_end) + timedelta(days=1)).strftime(
+                                    settings.JIRA_API_DATE_FORMAT
+                                ),
+                            ),
                         ):
-                            # Special cases for partial day when the sprint starts/ends.
-                            if vacation_date == self.future_sprint_start:
-                                row.vacation_time += \
-                                    (self.commitments[row.user.name]['days'][vacation_date] - vacation['seconds']) * \
-                                    (1 - self.sprint_division[row.user.displayName])
-                            elif vacation_date == self.future_sprint_end:
-                                row.vacation_time += \
-                                    (self.commitments[row.user.name]['days'][vacation_date] - vacation['seconds']) * \
-                                    self.sprint_division[row.user.displayName]
-                            else:
-                                row.vacation_time += \
-                                    (self.commitments[row.user.name]['days'][vacation_date] - vacation['seconds'])
-                    elif row.user.displayName < vacation['user']:
+                            row.vacation_time += self._get_vacation_for_day(
+                                self.commitments[row.user.name]["days"][vacation_date],
+                                vacation_date,
+                                vacation["seconds"],
+                                row.user.displayName,
+                            )
+                    elif row.user.displayName < vacation["user"]:
                         # Small optimization, as users' vacations are sorted.
                         break
 
+                # Remove the "padding" from a day before and after the sprint.
                 # noinspection PyTypeChecker
-                row.set_goal_time(self.commitments[row.user.name]['total'] - row.vacation_time)
+                row.set_goal_time(
+                    self.commitments[row.user.name]["total"]
+                    - self.commitments[row.user.name]["days"][self.before_future_sprint_start]
+                    - self.commitments[row.user.name]["days"][self.after_future_sprint_end]
+                    - row.vacation_time
+                )
+
+    def _get_vacation_for_day(self, commitments: int, date: str, planned_commitments: int, username: str) -> float:
+        """
+        Returns vacation time for specific users during a day.
+
+        Because of the timezones we need to consider 4 edge cases. When vacations are scheduled:
+        1. For the last day of the active sprint, there are two subcases:
+            a. Positive timezone - this day is completely a part of the active sprint, so this time is ignored (0).
+            b. Negative timezone - this day can span the active and next sprint, because user can work after the sprint
+               ends. The ratio is represented by `1 - division`.
+        2. For the first day of the next sprint, there are two subcases:
+            a. Positive timezone - this day can span the active and next sprint, because user can work after the sprint
+               ends. The ratio is represented by `1 - division`.
+            b. Negative timezone - this day is completely a part of the next sprint, so it is counted as vacations.
+        3. For the last day of the next sprint, there are two subcases:
+            a. Positive timezone - this day is completely a part of the next sprint, so it is counted as vacations.
+            b. Negative timezone - this day can span the next and future next sprint, because user can work after
+               the sprint ends. The ratio is represented by `division`.
+        4. For the first day of the future next sprint, there are two subcases:
+            a. Positive timezone - this day can span the next and future next sprint, because user can work after
+               the sprint ends. The ratio is represented by `division`.
+            b. Negative timezone - this day is completely a part of the future next sprint, so this time is ignored (0).
+
+        `division` - a part of the user's availability before the start of the sprint.
+
+        TODO: Check whether this works correctly with other sprint start times than midnight UTC.
+              For these it can span 3 days, so we might have 6 (or even more) corner cases.
+        """
+        division, positive_timezone = self.sprint_division[username]
+        vacations = commitments - planned_commitments
+
+        if date < self.future_sprint_start:
+            return vacations * (1 - division) if not positive_timezone else 0
+        elif date == self.future_sprint_start:
+            return vacations * (1 - division) if positive_timezone else vacations
+        elif date == self.future_sprint_end:
+            return vacations * division if not positive_timezone else vacations
+        elif date > self.future_sprint_end:
+            return vacations * division if positive_timezone else 0
+
+        return vacations

sprints/dashboard/tests/test_models.py

@@ -1,7 +1,10 @@
 import pytest
 from django.conf import settings
 
-from sprints.dashboard.models import DashboardIssue
+from sprints.dashboard.models import (
+    Dashboard,
+    DashboardIssue,
+)
 from sprints.dashboard.tests.helpers import does_not_raise
 
 
@@ -94,9 +97,43 @@
         ),
     ],
 )
-def test_get_sprint_meeting_day_division_for_member(test_description, test_directive, expected, raises):
+def test_get_bot_directive(test_description, test_directive, expected, raises):
     mock_issue = object.__new__(DashboardIssue)
     mock_issue.description = test_description
 
     with raises:
         assert mock_issue.get_bot_directive(test_directive) == expected
+
+
+@pytest.mark.parametrize(
+    "commitments, date, planned_commitments, username, division, expected",
+    [
+        # For positive timezone the day before sprint starts is within the previous sprint, so division is ignored.
+        (28800, "2020-11-16", 28700, "x", {"x": (0.6, True)}, 0),
+        # For negative timezone the day before sprint can span two sprints...
+        (28800, "2020-11-16", 28700, "x", {"x": (0.4, False)}, 60),
+        # ...but it does not need to do so.
+        (28800, "2020-11-16", 28700, "x", {"x": (0, False)}, 100),
+        # For the positive timezone the first day of the sprint may span two sprints.
+        (28800, "2020-11-17", 28700, "x", {"x": (0.6, True)}, 40),
+        # For the negative timezone the first day of the sprint is only in a single sprint.
+        (28800, "2020-11-17", 28700, "x", {"x": (0.4, False)}, 100),
+        # For the positive timezone the last day of the sprint is only in a single sprint.
+        (28800, "2020-11-30", 28700, "x", {"x": (0.6, True)}, 100),
+        # For the negative timezone the last day of the sprint can span two sprints.
+        (28800, "2020-11-30", 28700, "x", {"x": (0.4, False)}, 40),
+        # For the positive timezone the day after last day of the sprint can span two sprints...
+        (28800, "2020-12-1", 28700, "x", {"x": (0.6, True)}, 60),
+        # ... but it does not need to do so.
+        (28800, "2020-12-1", 28700, "x", {"x": (0, True)}, 0),
+        # For the negative timezone the last day of the sprint is only in a single sprint.
+        (28800, "2020-12-1", 28700, "x", {"x": (0.4, False)}, 0),
+    ],
+)
+def test_get_vacation_for_day(commitments, date, planned_commitments, username, division, expected):
+    mock_dashboard = object.__new__(Dashboard)
+    mock_dashboard.future_sprint_start = "2020-11-17"
+    mock_dashboard.future_sprint_end = "2020-11-30"
+    mock_dashboard.sprint_division = division
+
+    assert mock_dashboard._get_vacation_for_day(commitments, date, planned_commitments, username) == expected

sprints/dashboard/tests/test_utils.py

@@ -1,4 +1,5 @@
 import re
+from unittest.mock import patch
 
 import pytest
 from django.conf import settings
@@ -306,17 +307,18 @@ def test_column_number_to_excel(test_input, expected):
     assert _column_number_to_excel(test_input) == expected
 
 
+@override_settings(DEBUG=True)  # `sentry_sdk` does not capture exceptions in `DEBUG` mode.
 @pytest.mark.parametrize(
-    "test_input, expected", [
-        ('12pm-9pm*', .44),
-        ('3pm-12am-', .11),
-        ('9am - 5pm*', .88),
-        ('6pm-1am*-', 0.),
-        ('11pm-8am*', 1.),
-        ('2pm-5:30pm', .67),
-        ('8am-4pm', 1.),
-        ('invalid', 0.),
+    "hours, expected",
+    [
+        ("12pm-9pm*", 0),
+        ("3pm-12am", 0),
+        ("12am-2am", 0),
+        ("3pm - 1am", 0.9),
+        ("11:30pm-2am", 0.2),
+        ("invalid", 0),
     ],
 )
-def test_get_sprint_meeting_day_division_for_member(test_input, expected):
-    assert _get_sprint_meeting_day_division_for_member(test_input) == pytest.approx(expected, 0.1)
+def test_get_sprint_meeting_day_division_for_member(hours, expected):
+    sprint_start = "2020-01-01"
+    assert _get_sprint_meeting_day_division_for_member(hours, sprint_start) == pytest.approx(expected, 0.1)