Fix parsing of row_per_reading format with malformed dates

app/models/amr_data_feed_config.rb

@@ -11,6 +11,7 @@
 #  description             :text             not null
 #  enabled                 :boolean          default(TRUE), not null
 #  expected_units          :string
+#  half_hourly_labelling   :enum
 #  handle_off_by_one       :boolean          default(FALSE)
 #  header_example          :text
 #  id                      :bigint(8)        not null, primary key
@@ -57,8 +58,17 @@ class AmrDataFeedConfig < ApplicationRecord
   validates :identifier, :description, uniqueness: true
   validates_presence_of :identifier, :description
 
+  validates :row_per_reading, inclusion: [true], if: :positional_index
+  validate :period_or_time_field, if: :positional_index
+
+  validates_presence_of :msn_field, if: :lookup_by_serial_number
+
   BLANK_THRESHOLD = 1
 
+  def period_or_time_field
+    errors.add(:base, 'Must specify either period or time field') if positional_index && reading_time_field.blank? && period_field.blank?
+  end
+
   def map_of_fields_to_indexes(header = nil)
     this_header = header || header_example
     header_array = this_header.split(',')

app/services/amr/single_read_converter.rb

@@ -7,13 +7,12 @@ class InvalidTimeStringError < StandardError; end
     def initialize(amr_data_feed_config, single_reading_array)
       @amr_data_feed_config = amr_data_feed_config
       @single_reading_array = single_reading_array
-      @indexed = @amr_data_feed_config[:positional_index]
       @results_array = []
     end
 
     # Reading will be in one of the following formats:
     #
-    # * With timestamps starting at 00:00:00 or 00:30:00 e.g.:
+    # * With timestamps, which may be based on labelling either the start or the end of the half-hourly period
     #     {:amr_data_feed_config_id=>6, :mpan_mprn=>"1710035168313", :reading_date=>"26 Aug 2019 00:30:00", :readings=>["14.4"]
     #
     # * With `indexed: true` and reading date and reading time split to 2 fields with timestamps starting at 00:00:00 or 00:30:00 e.g.:
@@ -24,32 +23,32 @@ def initialize(amr_data_feed_config, single_reading_array)
     #     {:amr_data_feed_config_id=>6, :mpan_mprn=>"1710035168313", :reading_date=>"26 Aug 2019", :period=>2, :readings=>["14.4"]
     #
     # ...where each consecutive reading is a new HH period
-    # For here we need to determine the period by counting the index into the array
+    #
+    # The bulk of the code here is about building an array of x48 readings from each individual readings,
+    # ensuring the right array indexes are used based on correctly interpreting the above. Internally we label
+    # HH periods from the start of the half hour.
+    #
+    # Mapping from numbers periods is simple, for the others formats we have to interpret the time or timestamps correctly
     def perform
-      @single_reading_array.each do |single_reading|
+      @single_reading_array.each do |reading|
         # ignore rows that dont have necessary information
-        next unless single_reading[:reading_date].present? && single_reading[:mpan_mprn].present?
+        next unless reading[:reading_date].present? && reading[:mpan_mprn].present?
 
-        reading_day = Date.parse(single_reading[:reading_date])
+        reading_date = parse_reading_date(reading)
 
-        reading = single_reading[:readings].first.to_f
+        kwh = reading[:readings].first.to_f
 
-        reading_index = reading_index_of_record(reading_day, single_reading)
+        reading_index = reading_index_of_record(reading)
         next if reading_index.nil?
 
-        if last_reading_of_day?(reading_index)
-          reading_day = reading_day - 1.day
-          reading_index = 47
-        end
-
-        this_day = day_from_results(reading_day, single_reading[:mpan_mprn])
+        this_day = day_from_results(reading_date, reading[:mpan_mprn])
 
         if this_day.present?
-          this_day[:readings][reading_index] = reading
+          this_day[:readings][reading_index] = kwh
         else
           readings = Array.new(48)
-          readings[reading_index] = reading
-          new_record = { reading_date: reading_day, readings: readings, mpan_mprn: single_reading[:mpan_mprn], amr_data_feed_config_id: single_reading[:amr_data_feed_config_id], meter_id: single_reading[:meter_id] }
+          readings[reading_index] = kwh
+          new_record = { reading_date:, readings:, mpan_mprn: reading[:mpan_mprn], amr_data_feed_config_id: reading[:amr_data_feed_config_id], meter_id: reading[:meter_id] }
           @results_array << new_record
         end
       end
@@ -93,33 +92,69 @@ def reject_any_low_reading_days
       @results_array.reject { |result| result[:readings].count(&:blank?) > @amr_data_feed_config.blank_threshold }
     end
 
-    def last_reading_of_day?(reading_index)
-      reading_index == -1
+    def day_from_results(reading_day, mpan_mprn)
+      @results_array.find { |result| result[:reading_date] == reading_day && result[:mpan_mprn] == mpan_mprn }
     end
 
-    def reading_index_of_record(reading_day, single_reading)
-      if @indexed
-        reading_row_index_for(single_reading)
+    # Parses the reading date. When using a timestamp with readings labelled at the end of the half-hourly period
+    # we need to adjust the date as 2024-10-10T00:00:00Z should be usage from 2024-10-09 23:30.
+    def parse_reading_date(reading)
+      if indexed_with_period? || indexed_by_time? || half_hourly_labelling_at_start?
+        Date.parse(reading[:reading_date])
       else
-        reading_day_time_for(reading_day, single_reading)
+        reading_day_time = Time.zone.parse(reading[:reading_date])
+        raise Date::Error.new(reading[:reading_date]) if reading_day_time.nil?
+        # roll the date backward for last reading of day
+        reading_day_time == reading_day_time.midnight ? (reading_day_time - 1.day).to_date : reading_day_time.to_date
       end
     end
 
-    def day_from_results(reading_day, mpan_mprn)
-      @results_array.find { |result| result[:reading_date] == reading_day && result[:mpan_mprn] == mpan_mprn }
+    # Find array index for this reading
+    def reading_index_of_record(reading)
+      if indexed_with_period?
+        index_from_period(reading)
+      elsif indexed_by_time?
+        index_from_time_field(reading)
+      elsif half_hourly_labelling_at_start?
+        index_from_timestamps_at_start_of_half_hour(reading)
+      else
+        index_from_timestamps_at_end_of_half_hour(reading)
+      end
     end
 
-    def reading_day_time_for(reading_day, single_reading)
-      reading_day_time = Time.parse(single_reading[:reading_date]).utc
-      first_reading_time = Time.parse(reading_day.strftime('%Y-%m-%d')).utc + 30.minutes
+    def indexed_with_period?
+      @amr_data_feed_config[:positional_index] && @amr_data_feed_config[:period_field]
+    end
+
+    def indexed_by_time?
+      @amr_data_feed_config[:positional_index] && @amr_data_feed_config[:reading_time_field]
+    end
+
+    def half_hourly_labelling_at_start?
+      @amr_data_feed_config.half_hourly_labelling&.to_sym == :start
+    end
 
-      ((reading_day_time - first_reading_time) / 30.minutes).to_i
+    # Periods are numbered 1-48
+    def index_from_period(reading)
+      return reading[:period].to_i - 1
     end
 
-    def reading_row_index_for(single_reading)
-      return single_reading[:period].to_i - 1 if single_reading[:period]
+    # Reformat the reading time into %H:%M format and calculate index
+    def index_from_time_field(reading)
+      time_string = SingleReadConverter.convert_time_string_to_usable_time(reading[:reading_time])
+      TimeOfDay.parse(time_string).to_halfhour_index
+    end
+
+    # Parse the reading time stamp using configured format, then extract just the time to calculate index
+    def index_from_timestamps_at_start_of_half_hour(reading)
+      reading_day_time = Time.strptime(reading[:reading_date], @amr_data_feed_config.date_format)
+      time_string = reading_day_time.strftime('%H:%M')
+      TimeOfDay.parse(time_string).to_halfhour_index
+    end
 
-      time_string = SingleReadConverter.convert_time_string_to_usable_time(single_reading[:reading_time])
+    def index_from_timestamps_at_end_of_half_hour(reading)
+      reading_day_time = Time.zone.parse(reading[:reading_date])
+      time_string = reading_day_time == reading_day_time.midnight ? '23:30' : reading_day_time.advance(minutes: -30).strftime('%H:%M')
       TimeOfDay.parse(time_string).to_halfhour_index
     end
   end

db/migrate/20241010131742_add_half_hourly_labelling_to_data_feed_config.rb

@@ -0,0 +1,11 @@
+class AddHalfHourlyLabellingToDataFeedConfig < ActiveRecord::Migration[7.1]
+  def up
+    create_enum :half_hourly_labelling, ["start", "end"]
+    add_column :amr_data_feed_configs, :half_hourly_labelling, :enum, enum_type: :half_hourly_labelling, default: nil, null: true
+  end
+
+  def down
+    remove_column :amr_data_feed_configs, :half_hourly_labelling
+    drop_enum :half_hourly_labelling
+  end
+end