diff --git a/app/models/amr_data_feed_config.rb b/app/models/amr_data_feed_config.rb index 65bc11e5a1..670bac4c43 100644 --- a/app/models/amr_data_feed_config.rb +++ b/app/models/amr_data_feed_config.rb @@ -11,6 +11,7 @@ # description :text not null # enabled :boolean default(TRUE), not null # expected_units :string +# half_hourly_labelling :enum # handle_off_by_one :boolean default(FALSE) # header_example :text # id :bigint(8) not null, primary key @@ -57,8 +58,17 @@ class AmrDataFeedConfig < ApplicationRecord validates :identifier, :description, uniqueness: true validates_presence_of :identifier, :description + validates :row_per_reading, inclusion: [true], if: :positional_index + validate :period_or_time_field, if: :positional_index + + validates_presence_of :msn_field, if: :lookup_by_serial_number + BLANK_THRESHOLD = 1 + def period_or_time_field + errors.add(:base, 'Must specify either period or time field') if positional_index && reading_time_field.blank? && period_field.blank? + end + def map_of_fields_to_indexes(header = nil) this_header = header || header_example header_array = this_header.split(',') diff --git a/app/services/amr/single_read_converter.rb b/app/services/amr/single_read_converter.rb index bba676be4d..838b73c522 100644 --- a/app/services/amr/single_read_converter.rb +++ b/app/services/amr/single_read_converter.rb @@ -7,13 +7,12 @@ class InvalidTimeStringError < StandardError; end def initialize(amr_data_feed_config, single_reading_array) @amr_data_feed_config = amr_data_feed_config @single_reading_array = single_reading_array - @indexed = @amr_data_feed_config[:positional_index] @results_array = [] end # Reading will be in one of the following formats: # - # * With timestamps starting at 00:00:00 or 00:30:00 e.g.: + # * With timestamps, which may be based on labelling either the start or the end of the half-hourly period # {:amr_data_feed_config_id=>6, :mpan_mprn=>"1710035168313", :reading_date=>"26 Aug 2019 00:30:00", :readings=>["14.4"] # # * With `indexed: true` and reading date and reading time split to 2 fields with timestamps starting at 00:00:00 or 00:30:00 e.g.: @@ -24,32 +23,32 @@ def initialize(amr_data_feed_config, single_reading_array) # {:amr_data_feed_config_id=>6, :mpan_mprn=>"1710035168313", :reading_date=>"26 Aug 2019", :period=>2, :readings=>["14.4"] # # ...where each consecutive reading is a new HH period - # For here we need to determine the period by counting the index into the array + # + # The bulk of the code here is about building an array of x48 readings from each individual readings, + # ensuring the right array indexes are used based on correctly interpreting the above. Internally we label + # HH periods from the start of the half hour. + # + # Mapping from numbers periods is simple, for the others formats we have to interpret the time or timestamps correctly def perform - @single_reading_array.each do |single_reading| + @single_reading_array.each do |reading| # ignore rows that dont have necessary information - next unless single_reading[:reading_date].present? && single_reading[:mpan_mprn].present? + next unless reading[:reading_date].present? && reading[:mpan_mprn].present? - reading_day = Date.parse(single_reading[:reading_date]) + reading_date = parse_reading_date(reading) - reading = single_reading[:readings].first.to_f + kwh = reading[:readings].first.to_f - reading_index = reading_index_of_record(reading_day, single_reading) + reading_index = reading_index_of_record(reading) next if reading_index.nil? - if last_reading_of_day?(reading_index) - reading_day = reading_day - 1.day - reading_index = 47 - end - - this_day = day_from_results(reading_day, single_reading[:mpan_mprn]) + this_day = day_from_results(reading_date, reading[:mpan_mprn]) if this_day.present? - this_day[:readings][reading_index] = reading + this_day[:readings][reading_index] = kwh else readings = Array.new(48) - readings[reading_index] = reading - new_record = { reading_date: reading_day, readings: readings, mpan_mprn: single_reading[:mpan_mprn], amr_data_feed_config_id: single_reading[:amr_data_feed_config_id], meter_id: single_reading[:meter_id] } + readings[reading_index] = kwh + new_record = { reading_date:, readings:, mpan_mprn: reading[:mpan_mprn], amr_data_feed_config_id: reading[:amr_data_feed_config_id], meter_id: reading[:meter_id] } @results_array << new_record end end @@ -93,33 +92,69 @@ def reject_any_low_reading_days @results_array.reject { |result| result[:readings].count(&:blank?) > @amr_data_feed_config.blank_threshold } end - def last_reading_of_day?(reading_index) - reading_index == -1 + def day_from_results(reading_day, mpan_mprn) + @results_array.find { |result| result[:reading_date] == reading_day && result[:mpan_mprn] == mpan_mprn } end - def reading_index_of_record(reading_day, single_reading) - if @indexed - reading_row_index_for(single_reading) + # Parses the reading date. When using a timestamp with readings labelled at the end of the half-hourly period + # we need to adjust the date as 2024-10-10T00:00:00Z should be usage from 2024-10-09 23:30. + def parse_reading_date(reading) + if indexed_with_period? || indexed_by_time? || half_hourly_labelling_at_start? + Date.parse(reading[:reading_date]) else - reading_day_time_for(reading_day, single_reading) + reading_day_time = Time.zone.parse(reading[:reading_date]) + raise Date::Error.new(reading[:reading_date]) if reading_day_time.nil? + # roll the date backward for last reading of day + reading_day_time == reading_day_time.midnight ? (reading_day_time - 1.day).to_date : reading_day_time.to_date end end - def day_from_results(reading_day, mpan_mprn) - @results_array.find { |result| result[:reading_date] == reading_day && result[:mpan_mprn] == mpan_mprn } + # Find array index for this reading + def reading_index_of_record(reading) + if indexed_with_period? + index_from_period(reading) + elsif indexed_by_time? + index_from_time_field(reading) + elsif half_hourly_labelling_at_start? + index_from_timestamps_at_start_of_half_hour(reading) + else + index_from_timestamps_at_end_of_half_hour(reading) + end end - def reading_day_time_for(reading_day, single_reading) - reading_day_time = Time.parse(single_reading[:reading_date]).utc - first_reading_time = Time.parse(reading_day.strftime('%Y-%m-%d')).utc + 30.minutes + def indexed_with_period? + @amr_data_feed_config[:positional_index] && @amr_data_feed_config[:period_field] + end + + def indexed_by_time? + @amr_data_feed_config[:positional_index] && @amr_data_feed_config[:reading_time_field] + end + + def half_hourly_labelling_at_start? + @amr_data_feed_config.half_hourly_labelling&.to_sym == :start + end - ((reading_day_time - first_reading_time) / 30.minutes).to_i + # Periods are numbered 1-48 + def index_from_period(reading) + return reading[:period].to_i - 1 end - def reading_row_index_for(single_reading) - return single_reading[:period].to_i - 1 if single_reading[:period] + # Reformat the reading time into %H:%M format and calculate index + def index_from_time_field(reading) + time_string = SingleReadConverter.convert_time_string_to_usable_time(reading[:reading_time]) + TimeOfDay.parse(time_string).to_halfhour_index + end + + # Parse the reading time stamp using configured format, then extract just the time to calculate index + def index_from_timestamps_at_start_of_half_hour(reading) + reading_day_time = Time.strptime(reading[:reading_date], @amr_data_feed_config.date_format) + time_string = reading_day_time.strftime('%H:%M') + TimeOfDay.parse(time_string).to_halfhour_index + end - time_string = SingleReadConverter.convert_time_string_to_usable_time(single_reading[:reading_time]) + def index_from_timestamps_at_end_of_half_hour(reading) + reading_day_time = Time.zone.parse(reading[:reading_date]) + time_string = reading_day_time == reading_day_time.midnight ? '23:30' : reading_day_time.advance(minutes: -30).strftime('%H:%M') TimeOfDay.parse(time_string).to_halfhour_index end end diff --git a/db/migrate/20241010131742_add_half_hourly_labelling_to_data_feed_config.rb b/db/migrate/20241010131742_add_half_hourly_labelling_to_data_feed_config.rb new file mode 100644 index 0000000000..4c7138d700 --- /dev/null +++ b/db/migrate/20241010131742_add_half_hourly_labelling_to_data_feed_config.rb @@ -0,0 +1,11 @@ +class AddHalfHourlyLabellingToDataFeedConfig < ActiveRecord::Migration[7.1] + def up + create_enum :half_hourly_labelling, ["start", "end"] + add_column :amr_data_feed_configs, :half_hourly_labelling, :enum, enum_type: :half_hourly_labelling, default: nil, null: true + end + + def down + remove_column :amr_data_feed_configs, :half_hourly_labelling + drop_enum :half_hourly_labelling + end +end diff --git a/db/schema.rb b/db/schema.rb index 796a1ace6e..adc11014a4 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_10_07_110615) do +ActiveRecord::Schema[7.1].define(version: 2024_10_10_131742) do # These are extensions that must be enabled in order to support this database enable_extension "hstore" enable_extension "pgcrypto" @@ -20,6 +20,7 @@ # Note that some types may not work with other database engines. Be careful if changing database. create_enum "data_sharing", ["public", "within_group", "private"] create_enum "dcc_meter", ["no", "smets2", "other"] + create_enum "half_hourly_labelling", ["start", "end"] create_table "academic_years", force: :cascade do |t| t.date "start_date" @@ -416,6 +417,7 @@ t.text "reading_time_field" t.boolean "convert_to_kwh", default: false t.boolean "delayed_reading", default: false, null: false + t.enum "half_hourly_labelling", enum_type: "half_hourly_labelling" t.index ["description"], name: "index_amr_data_feed_configs_on_description", unique: true t.index ["identifier"], name: "index_amr_data_feed_configs_on_identifier", unique: true end @@ -1546,8 +1548,8 @@ t.boolean "school_will_be_public", default: true t.integer "default_chart_preference", default: 0, null: false t.integer "country", default: 0, null: false - t.enum "data_sharing", default: "public", null: false, enum_type: "data_sharing" t.bigint "funder_id" + t.enum "data_sharing", default: "public", null: false, enum_type: "data_sharing" t.index ["created_by_id"], name: "index_school_onboardings_on_created_by_id" t.index ["created_user_id"], name: "index_school_onboardings_on_created_user_id" t.index ["funder_id"], name: "index_school_onboardings_on_funder_id" @@ -3087,6 +3089,116 @@ ORDER BY alert_generation_runs.school_id, alert_generation_runs.created_at DESC) latest_runs WHERE ((energy.alert_generation_run_id = latest_runs.id) AND (storage.alert_generation_run_id = latest_runs.id)); SQL + create_view "configurable_periods", sql_definition: <<-SQL + WITH electricity AS ( + SELECT alerts.alert_generation_run_id, + alerts.comparison_report_id, + json.current_period_kwh, + json.previous_period_kwh, + json.current_period_co2, + json.previous_period_co2, + json.current_period_gbp, + json.previous_period_gbp, + json.tariff_has_changed, + json.pupils_changed, + json.floor_area_changed + FROM (alerts + JOIN alert_types ON ((alerts.alert_type_id = alert_types.id))), + LATERAL jsonb_to_record(alerts.variables) json(current_period_kwh double precision, previous_period_kwh double precision, current_period_co2 double precision, previous_period_co2 double precision, current_period_gbp double precision, previous_period_gbp double precision, tariff_has_changed boolean, pupils_changed boolean, floor_area_changed boolean) + WHERE (alert_types.class_name = 'AlertConfigurablePeriodElectricityComparison'::text) + ), gas AS ( + SELECT alerts.alert_generation_run_id, + alerts.comparison_report_id, + json.current_period_kwh, + json.previous_period_kwh, + json.current_period_co2, + json.previous_period_co2, + json.current_period_gbp, + json.previous_period_gbp, + json.previous_period_kwh_unadjusted, + json.tariff_has_changed, + json.pupils_changed, + json.floor_area_changed + FROM (alerts + JOIN alert_types ON ((alerts.alert_type_id = alert_types.id))), + LATERAL jsonb_to_record(alerts.variables) json(current_period_kwh double precision, previous_period_kwh double precision, current_period_co2 double precision, previous_period_co2 double precision, current_period_gbp double precision, previous_period_gbp double precision, previous_period_kwh_unadjusted double precision, tariff_has_changed boolean, pupils_changed boolean, floor_area_changed boolean) + WHERE (alert_types.class_name = 'AlertConfigurablePeriodGasComparison'::text) + ), storage_heater AS ( + SELECT alerts.alert_generation_run_id, + alerts.comparison_report_id, + json.current_period_kwh, + json.previous_period_kwh, + json.current_period_co2, + json.previous_period_co2, + json.current_period_gbp, + json.previous_period_gbp, + json.previous_period_kwh_unadjusted, + json.tariff_has_changed, + json.pupils_changed, + json.floor_area_changed + FROM (alerts + JOIN alert_types ON ((alerts.alert_type_id = alert_types.id))), + LATERAL jsonb_to_record(alerts.variables) json(current_period_kwh double precision, previous_period_kwh double precision, current_period_co2 double precision, previous_period_co2 double precision, current_period_gbp double precision, previous_period_gbp double precision, previous_period_kwh_unadjusted double precision, tariff_has_changed boolean, pupils_changed boolean, floor_area_changed boolean) + WHERE (alert_types.class_name = 'AlertConfigurablePeriodStorageHeaterComparison'::text) + ), benchmark AS ( + SELECT alerts.alert_generation_run_id, + data.solar_type + FROM (alerts + JOIN alert_types ON ((alerts.alert_type_id = alert_types.id))), + LATERAL jsonb_to_record(alerts.variables) data(solar_type text) + WHERE (alert_types.class_name = 'AlertEnergyAnnualVersusBenchmark'::text) + ), additional AS ( + SELECT alerts.alert_generation_run_id, + alerts.school_id, + data.activation_date + FROM (alerts + JOIN alert_types ON ((alerts.alert_type_id = alert_types.id))), + LATERAL jsonb_to_record(alerts.variables) data(activation_date date) + WHERE (alert_types.class_name = 'AlertAdditionalPrioritisationData'::text) + ), latest_runs AS ( + SELECT ranked.id + FROM ( SELECT alert_generation_runs.id, + row_number() OVER (PARTITION BY alert_generation_runs.school_id ORDER BY alert_generation_runs.created_at DESC) AS row_num + FROM alert_generation_runs) ranked + WHERE (ranked.row_num = 1) + ) + SELECT latest_runs.id, + additional.school_id, + additional.activation_date, + benchmark.solar_type, + electricity.current_period_kwh AS electricity_current_period_kwh, + electricity.previous_period_kwh AS electricity_previous_period_kwh, + electricity.current_period_co2 AS electricity_current_period_co2, + electricity.previous_period_co2 AS electricity_previous_period_co2, + electricity.current_period_gbp AS electricity_current_period_gbp, + electricity.previous_period_gbp AS electricity_previous_period_gbp, + electricity.tariff_has_changed AS electricity_tariff_has_changed, + gas.current_period_kwh AS gas_current_period_kwh, + gas.previous_period_kwh AS gas_previous_period_kwh, + gas.current_period_co2 AS gas_current_period_co2, + gas.previous_period_co2 AS gas_previous_period_co2, + gas.current_period_gbp AS gas_current_period_gbp, + gas.previous_period_gbp AS gas_previous_period_gbp, + gas.previous_period_kwh_unadjusted AS gas_previous_period_kwh_unadjusted, + gas.tariff_has_changed AS gas_tariff_has_changed, + storage_heater.current_period_kwh AS storage_heater_current_period_kwh, + storage_heater.previous_period_kwh AS storage_heater_previous_period_kwh, + storage_heater.current_period_co2 AS storage_heater_current_period_co2, + storage_heater.previous_period_co2 AS storage_heater_previous_period_co2, + storage_heater.current_period_gbp AS storage_heater_current_period_gbp, + storage_heater.previous_period_gbp AS storage_heater_previous_period_gbp, + storage_heater.previous_period_kwh_unadjusted AS storage_heater_previous_period_kwh_unadjusted, + storage_heater.tariff_has_changed AS storage_heater_tariff_has_changed, + COALESCE(electricity.comparison_report_id, gas.comparison_report_id, storage_heater.comparison_report_id) AS comparison_report_id, + (electricity.pupils_changed OR gas.pupils_changed OR storage_heater.pupils_changed) AS pupils_changed, + (electricity.floor_area_changed OR gas.floor_area_changed OR storage_heater.floor_area_changed) AS floor_area_changed + FROM (((((latest_runs + JOIN additional ON ((latest_runs.id = additional.alert_generation_run_id))) + LEFT JOIN benchmark ON ((latest_runs.id = benchmark.alert_generation_run_id))) + LEFT JOIN electricity ON ((latest_runs.id = electricity.alert_generation_run_id))) + LEFT JOIN gas ON (((latest_runs.id = gas.alert_generation_run_id) AND ((electricity.comparison_report_id IS NULL) OR (electricity.comparison_report_id = gas.comparison_report_id))))) + LEFT JOIN storage_heater ON (((latest_runs.id = storage_heater.alert_generation_run_id) AND ((gas.comparison_report_id IS NULL) OR (gas.comparison_report_id = storage_heater.comparison_report_id)) AND ((electricity.comparison_report_id IS NULL) OR (electricity.comparison_report_id = storage_heater.comparison_report_id))))); + SQL create_view "electricity_consumption_during_holidays", sql_definition: <<-SQL SELECT latest_runs.id, data.alert_generation_run_id, @@ -3398,6 +3510,134 @@ LEFT JOIN gas ON ((latest_runs.id = gas.alert_generation_run_id))) LEFT JOIN hot_water ON ((latest_runs.id = hot_water.alert_generation_run_id))); SQL + create_view "holiday_and_terms", sql_definition: <<-SQL + WITH electricity AS ( + SELECT alerts.alert_generation_run_id, + json.current_period_kwh, + json.previous_period_kwh, + json.current_period_co2, + json.previous_period_co2, + json.current_period_gbp, + json.previous_period_gbp, + json.tariff_has_changed, + json.pupils_changed, + json.floor_area_changed, + json.current_period_type, + json.current_period_start_date, + json.current_period_end_date, + json.truncated_current_period + FROM alerts, + alert_types, + LATERAL jsonb_to_record(alerts.variables) json(current_period_kwh double precision, previous_period_kwh double precision, current_period_co2 double precision, previous_period_co2 double precision, current_period_gbp double precision, previous_period_gbp double precision, tariff_has_changed boolean, pupils_changed boolean, floor_area_changed boolean, current_period_type text, current_period_start_date date, current_period_end_date date, truncated_current_period boolean) + WHERE ((alerts.alert_type_id = alert_types.id) AND (alert_types.class_name = 'AlertHolidayAndTermElectricityComparison'::text)) + ), gas AS ( + SELECT alerts.alert_generation_run_id, + json.current_period_kwh, + json.previous_period_kwh, + json.current_period_co2, + json.previous_period_co2, + json.current_period_gbp, + json.previous_period_gbp, + json.previous_period_kwh_unadjusted, + json.tariff_has_changed, + json.pupils_changed, + json.floor_area_changed, + json.current_period_type, + json.current_period_start_date, + json.current_period_end_date, + json.truncated_current_period + FROM alerts, + alert_types, + LATERAL jsonb_to_record(alerts.variables) json(current_period_kwh double precision, previous_period_kwh double precision, current_period_co2 double precision, previous_period_co2 double precision, current_period_gbp double precision, previous_period_gbp double precision, previous_period_kwh_unadjusted double precision, tariff_has_changed boolean, pupils_changed boolean, floor_area_changed boolean, current_period_type text, current_period_start_date date, current_period_end_date date, truncated_current_period boolean) + WHERE ((alerts.alert_type_id = alert_types.id) AND (alert_types.class_name = 'AlertHolidayAndTermGasComparison'::text)) + ), storage_heater AS ( + SELECT alerts.alert_generation_run_id, + json.current_period_kwh, + json.previous_period_kwh, + json.current_period_co2, + json.previous_period_co2, + json.current_period_gbp, + json.previous_period_gbp, + json.previous_period_kwh_unadjusted, + json.tariff_has_changed, + json.pupils_changed, + json.floor_area_changed, + json.current_period_type, + json.current_period_start_date, + json.current_period_end_date, + json.truncated_current_period + FROM alerts, + alert_types, + LATERAL jsonb_to_record(alerts.variables) json(current_period_kwh double precision, previous_period_kwh double precision, current_period_co2 double precision, previous_period_co2 double precision, current_period_gbp double precision, previous_period_gbp double precision, previous_period_kwh_unadjusted double precision, tariff_has_changed boolean, pupils_changed boolean, floor_area_changed boolean, current_period_type text, current_period_start_date date, current_period_end_date date, truncated_current_period boolean) + WHERE ((alerts.alert_type_id = alert_types.id) AND (alert_types.class_name = 'AlertHolidayAndTermStorageHeaterComparison'::text)) + ), enba AS ( + SELECT alerts.alert_generation_run_id, + data.solar_type + FROM alerts, + alert_types, + LATERAL jsonb_to_record(alerts.variables) data(solar_type text) + WHERE ((alerts.alert_type_id = alert_types.id) AND (alert_types.class_name = 'AlertEnergyAnnualVersusBenchmark'::text)) + ), additional AS ( + SELECT alerts.alert_generation_run_id, + alerts.school_id, + data.activation_date + FROM alerts, + alert_types, + LATERAL jsonb_to_record(alerts.variables) data(activation_date date) + WHERE ((alerts.alert_type_id = alert_types.id) AND (alert_types.class_name = 'AlertAdditionalPrioritisationData'::text)) + ), latest_runs AS ( + SELECT DISTINCT ON (alert_generation_runs.school_id) alert_generation_runs.id + FROM alert_generation_runs + ORDER BY alert_generation_runs.school_id, alert_generation_runs.created_at DESC + ) + SELECT latest_runs.id, + additional.school_id, + additional.activation_date, + (electricity.pupils_changed OR gas.pupils_changed OR storage_heater.pupils_changed) AS pupils_changed, + (electricity.floor_area_changed OR gas.floor_area_changed OR storage_heater.floor_area_changed) AS floor_area_changed, + enba.solar_type, + electricity.current_period_kwh AS electricity_current_period_kwh, + electricity.previous_period_kwh AS electricity_previous_period_kwh, + electricity.current_period_co2 AS electricity_current_period_co2, + electricity.previous_period_co2 AS electricity_previous_period_co2, + electricity.current_period_gbp AS electricity_current_period_gbp, + electricity.previous_period_gbp AS electricity_previous_period_gbp, + electricity.tariff_has_changed AS electricity_tariff_has_changed, + electricity.current_period_type AS electricity_current_period_type, + electricity.current_period_start_date AS electricity_current_period_start_date, + electricity.current_period_end_date AS electricity_current_period_end_date, + electricity.truncated_current_period AS electricity_truncated_current_period, + gas.current_period_kwh AS gas_current_period_kwh, + gas.previous_period_kwh AS gas_previous_period_kwh, + gas.current_period_co2 AS gas_current_period_co2, + gas.previous_period_co2 AS gas_previous_period_co2, + gas.current_period_gbp AS gas_current_period_gbp, + gas.previous_period_gbp AS gas_previous_period_gbp, + gas.previous_period_kwh_unadjusted AS gas_previous_period_kwh_unadjusted, + gas.tariff_has_changed AS gas_tariff_has_changed, + gas.current_period_type AS gas_current_period_type, + gas.current_period_start_date AS gas_current_period_start_date, + gas.current_period_end_date AS gas_current_period_end_date, + gas.truncated_current_period AS gas_truncated_current_period, + storage_heater.current_period_kwh AS storage_heater_current_period_kwh, + storage_heater.previous_period_kwh AS storage_heater_previous_period_kwh, + storage_heater.current_period_co2 AS storage_heater_current_period_co2, + storage_heater.previous_period_co2 AS storage_heater_previous_period_co2, + storage_heater.current_period_gbp AS storage_heater_current_period_gbp, + storage_heater.previous_period_gbp AS storage_heater_previous_period_gbp, + storage_heater.previous_period_kwh_unadjusted AS storage_heater_previous_period_kwh_unadjusted, + storage_heater.tariff_has_changed AS storage_heater_tariff_has_changed, + storage_heater.current_period_type AS storage_heater_current_period_type, + storage_heater.current_period_start_date AS storage_heater_current_period_start_date, + storage_heater.current_period_end_date AS storage_heater_current_period_end_date, + storage_heater.truncated_current_period AS storage_heater_truncated_current_period + FROM (((((latest_runs + JOIN additional ON ((latest_runs.id = additional.alert_generation_run_id))) + LEFT JOIN electricity ON ((latest_runs.id = electricity.alert_generation_run_id))) + LEFT JOIN gas ON ((latest_runs.id = gas.alert_generation_run_id))) + LEFT JOIN storage_heater ON ((latest_runs.id = storage_heater.alert_generation_run_id))) + LEFT JOIN enba ON ((latest_runs.id = enba.alert_generation_run_id))); + SQL create_view "holiday_usage_last_years", sql_definition: <<-SQL SELECT latest_runs.id, data.alert_generation_run_id, @@ -3686,244 +3926,6 @@ ORDER BY alert_generation_runs.school_id, alert_generation_runs.created_at DESC) latest_runs WHERE ((data.alert_generation_run_id = latest_runs.id) AND (additional.alert_generation_run_id = latest_runs.id)); SQL - create_view "holiday_and_terms", sql_definition: <<-SQL - WITH electricity AS ( - SELECT alerts.alert_generation_run_id, - json.current_period_kwh, - json.previous_period_kwh, - json.current_period_co2, - json.previous_period_co2, - json.current_period_gbp, - json.previous_period_gbp, - json.tariff_has_changed, - json.pupils_changed, - json.floor_area_changed, - json.current_period_type, - json.current_period_start_date, - json.current_period_end_date, - json.truncated_current_period - FROM alerts, - alert_types, - LATERAL jsonb_to_record(alerts.variables) json(current_period_kwh double precision, previous_period_kwh double precision, current_period_co2 double precision, previous_period_co2 double precision, current_period_gbp double precision, previous_period_gbp double precision, tariff_has_changed boolean, pupils_changed boolean, floor_area_changed boolean, current_period_type text, current_period_start_date date, current_period_end_date date, truncated_current_period boolean) - WHERE ((alerts.alert_type_id = alert_types.id) AND (alert_types.class_name = 'AlertHolidayAndTermElectricityComparison'::text)) - ), gas AS ( - SELECT alerts.alert_generation_run_id, - json.current_period_kwh, - json.previous_period_kwh, - json.current_period_co2, - json.previous_period_co2, - json.current_period_gbp, - json.previous_period_gbp, - json.previous_period_kwh_unadjusted, - json.tariff_has_changed, - json.pupils_changed, - json.floor_area_changed, - json.current_period_type, - json.current_period_start_date, - json.current_period_end_date, - json.truncated_current_period - FROM alerts, - alert_types, - LATERAL jsonb_to_record(alerts.variables) json(current_period_kwh double precision, previous_period_kwh double precision, current_period_co2 double precision, previous_period_co2 double precision, current_period_gbp double precision, previous_period_gbp double precision, previous_period_kwh_unadjusted double precision, tariff_has_changed boolean, pupils_changed boolean, floor_area_changed boolean, current_period_type text, current_period_start_date date, current_period_end_date date, truncated_current_period boolean) - WHERE ((alerts.alert_type_id = alert_types.id) AND (alert_types.class_name = 'AlertHolidayAndTermGasComparison'::text)) - ), storage_heater AS ( - SELECT alerts.alert_generation_run_id, - json.current_period_kwh, - json.previous_period_kwh, - json.current_period_co2, - json.previous_period_co2, - json.current_period_gbp, - json.previous_period_gbp, - json.previous_period_kwh_unadjusted, - json.tariff_has_changed, - json.pupils_changed, - json.floor_area_changed, - json.current_period_type, - json.current_period_start_date, - json.current_period_end_date, - json.truncated_current_period - FROM alerts, - alert_types, - LATERAL jsonb_to_record(alerts.variables) json(current_period_kwh double precision, previous_period_kwh double precision, current_period_co2 double precision, previous_period_co2 double precision, current_period_gbp double precision, previous_period_gbp double precision, previous_period_kwh_unadjusted double precision, tariff_has_changed boolean, pupils_changed boolean, floor_area_changed boolean, current_period_type text, current_period_start_date date, current_period_end_date date, truncated_current_period boolean) - WHERE ((alerts.alert_type_id = alert_types.id) AND (alert_types.class_name = 'AlertHolidayAndTermStorageHeaterComparison'::text)) - ), enba AS ( - SELECT alerts.alert_generation_run_id, - data.solar_type - FROM alerts, - alert_types, - LATERAL jsonb_to_record(alerts.variables) data(solar_type text) - WHERE ((alerts.alert_type_id = alert_types.id) AND (alert_types.class_name = 'AlertEnergyAnnualVersusBenchmark'::text)) - ), additional AS ( - SELECT alerts.alert_generation_run_id, - alerts.school_id, - data.activation_date - FROM alerts, - alert_types, - LATERAL jsonb_to_record(alerts.variables) data(activation_date date) - WHERE ((alerts.alert_type_id = alert_types.id) AND (alert_types.class_name = 'AlertAdditionalPrioritisationData'::text)) - ), latest_runs AS ( - SELECT DISTINCT ON (alert_generation_runs.school_id) alert_generation_runs.id - FROM alert_generation_runs - ORDER BY alert_generation_runs.school_id, alert_generation_runs.created_at DESC - ) - SELECT latest_runs.id, - additional.school_id, - additional.activation_date, - (electricity.pupils_changed OR gas.pupils_changed OR storage_heater.pupils_changed) AS pupils_changed, - (electricity.floor_area_changed OR gas.floor_area_changed OR storage_heater.floor_area_changed) AS floor_area_changed, - enba.solar_type, - electricity.current_period_kwh AS electricity_current_period_kwh, - electricity.previous_period_kwh AS electricity_previous_period_kwh, - electricity.current_period_co2 AS electricity_current_period_co2, - electricity.previous_period_co2 AS electricity_previous_period_co2, - electricity.current_period_gbp AS electricity_current_period_gbp, - electricity.previous_period_gbp AS electricity_previous_period_gbp, - electricity.tariff_has_changed AS electricity_tariff_has_changed, - electricity.current_period_type AS electricity_current_period_type, - electricity.current_period_start_date AS electricity_current_period_start_date, - electricity.current_period_end_date AS electricity_current_period_end_date, - electricity.truncated_current_period AS electricity_truncated_current_period, - gas.current_period_kwh AS gas_current_period_kwh, - gas.previous_period_kwh AS gas_previous_period_kwh, - gas.current_period_co2 AS gas_current_period_co2, - gas.previous_period_co2 AS gas_previous_period_co2, - gas.current_period_gbp AS gas_current_period_gbp, - gas.previous_period_gbp AS gas_previous_period_gbp, - gas.previous_period_kwh_unadjusted AS gas_previous_period_kwh_unadjusted, - gas.tariff_has_changed AS gas_tariff_has_changed, - gas.current_period_type AS gas_current_period_type, - gas.current_period_start_date AS gas_current_period_start_date, - gas.current_period_end_date AS gas_current_period_end_date, - gas.truncated_current_period AS gas_truncated_current_period, - storage_heater.current_period_kwh AS storage_heater_current_period_kwh, - storage_heater.previous_period_kwh AS storage_heater_previous_period_kwh, - storage_heater.current_period_co2 AS storage_heater_current_period_co2, - storage_heater.previous_period_co2 AS storage_heater_previous_period_co2, - storage_heater.current_period_gbp AS storage_heater_current_period_gbp, - storage_heater.previous_period_gbp AS storage_heater_previous_period_gbp, - storage_heater.previous_period_kwh_unadjusted AS storage_heater_previous_period_kwh_unadjusted, - storage_heater.tariff_has_changed AS storage_heater_tariff_has_changed, - storage_heater.current_period_type AS storage_heater_current_period_type, - storage_heater.current_period_start_date AS storage_heater_current_period_start_date, - storage_heater.current_period_end_date AS storage_heater_current_period_end_date, - storage_heater.truncated_current_period AS storage_heater_truncated_current_period - FROM (((((latest_runs - JOIN additional ON ((latest_runs.id = additional.alert_generation_run_id))) - LEFT JOIN electricity ON ((latest_runs.id = electricity.alert_generation_run_id))) - LEFT JOIN gas ON ((latest_runs.id = gas.alert_generation_run_id))) - LEFT JOIN storage_heater ON ((latest_runs.id = storage_heater.alert_generation_run_id))) - LEFT JOIN enba ON ((latest_runs.id = enba.alert_generation_run_id))); - SQL - create_view "configurable_periods", sql_definition: <<-SQL - WITH electricity AS ( - SELECT alerts.alert_generation_run_id, - alerts.comparison_report_id, - json.current_period_kwh, - json.previous_period_kwh, - json.current_period_co2, - json.previous_period_co2, - json.current_period_gbp, - json.previous_period_gbp, - json.tariff_has_changed, - json.pupils_changed, - json.floor_area_changed - FROM (alerts - JOIN alert_types ON ((alerts.alert_type_id = alert_types.id))), - LATERAL jsonb_to_record(alerts.variables) json(current_period_kwh double precision, previous_period_kwh double precision, current_period_co2 double precision, previous_period_co2 double precision, current_period_gbp double precision, previous_period_gbp double precision, tariff_has_changed boolean, pupils_changed boolean, floor_area_changed boolean) - WHERE (alert_types.class_name = 'AlertConfigurablePeriodElectricityComparison'::text) - ), gas AS ( - SELECT alerts.alert_generation_run_id, - alerts.comparison_report_id, - json.current_period_kwh, - json.previous_period_kwh, - json.current_period_co2, - json.previous_period_co2, - json.current_period_gbp, - json.previous_period_gbp, - json.previous_period_kwh_unadjusted, - json.tariff_has_changed, - json.pupils_changed, - json.floor_area_changed - FROM (alerts - JOIN alert_types ON ((alerts.alert_type_id = alert_types.id))), - LATERAL jsonb_to_record(alerts.variables) json(current_period_kwh double precision, previous_period_kwh double precision, current_period_co2 double precision, previous_period_co2 double precision, current_period_gbp double precision, previous_period_gbp double precision, previous_period_kwh_unadjusted double precision, tariff_has_changed boolean, pupils_changed boolean, floor_area_changed boolean) - WHERE (alert_types.class_name = 'AlertConfigurablePeriodGasComparison'::text) - ), storage_heater AS ( - SELECT alerts.alert_generation_run_id, - alerts.comparison_report_id, - json.current_period_kwh, - json.previous_period_kwh, - json.current_period_co2, - json.previous_period_co2, - json.current_period_gbp, - json.previous_period_gbp, - json.previous_period_kwh_unadjusted, - json.tariff_has_changed, - json.pupils_changed, - json.floor_area_changed - FROM (alerts - JOIN alert_types ON ((alerts.alert_type_id = alert_types.id))), - LATERAL jsonb_to_record(alerts.variables) json(current_period_kwh double precision, previous_period_kwh double precision, current_period_co2 double precision, previous_period_co2 double precision, current_period_gbp double precision, previous_period_gbp double precision, previous_period_kwh_unadjusted double precision, tariff_has_changed boolean, pupils_changed boolean, floor_area_changed boolean) - WHERE (alert_types.class_name = 'AlertConfigurablePeriodStorageHeaterComparison'::text) - ), benchmark AS ( - SELECT alerts.alert_generation_run_id, - data.solar_type - FROM (alerts - JOIN alert_types ON ((alerts.alert_type_id = alert_types.id))), - LATERAL jsonb_to_record(alerts.variables) data(solar_type text) - WHERE (alert_types.class_name = 'AlertEnergyAnnualVersusBenchmark'::text) - ), additional AS ( - SELECT alerts.alert_generation_run_id, - alerts.school_id, - data.activation_date - FROM (alerts - JOIN alert_types ON ((alerts.alert_type_id = alert_types.id))), - LATERAL jsonb_to_record(alerts.variables) data(activation_date date) - WHERE (alert_types.class_name = 'AlertAdditionalPrioritisationData'::text) - ), latest_runs AS ( - SELECT ranked.id - FROM ( SELECT alert_generation_runs.id, - row_number() OVER (PARTITION BY alert_generation_runs.school_id ORDER BY alert_generation_runs.created_at DESC) AS row_num - FROM alert_generation_runs) ranked - WHERE (ranked.row_num = 1) - ) - SELECT latest_runs.id, - additional.school_id, - additional.activation_date, - benchmark.solar_type, - electricity.current_period_kwh AS electricity_current_period_kwh, - electricity.previous_period_kwh AS electricity_previous_period_kwh, - electricity.current_period_co2 AS electricity_current_period_co2, - electricity.previous_period_co2 AS electricity_previous_period_co2, - electricity.current_period_gbp AS electricity_current_period_gbp, - electricity.previous_period_gbp AS electricity_previous_period_gbp, - electricity.tariff_has_changed AS electricity_tariff_has_changed, - gas.current_period_kwh AS gas_current_period_kwh, - gas.previous_period_kwh AS gas_previous_period_kwh, - gas.current_period_co2 AS gas_current_period_co2, - gas.previous_period_co2 AS gas_previous_period_co2, - gas.current_period_gbp AS gas_current_period_gbp, - gas.previous_period_gbp AS gas_previous_period_gbp, - gas.previous_period_kwh_unadjusted AS gas_previous_period_kwh_unadjusted, - gas.tariff_has_changed AS gas_tariff_has_changed, - storage_heater.current_period_kwh AS storage_heater_current_period_kwh, - storage_heater.previous_period_kwh AS storage_heater_previous_period_kwh, - storage_heater.current_period_co2 AS storage_heater_current_period_co2, - storage_heater.previous_period_co2 AS storage_heater_previous_period_co2, - storage_heater.current_period_gbp AS storage_heater_current_period_gbp, - storage_heater.previous_period_gbp AS storage_heater_previous_period_gbp, - storage_heater.previous_period_kwh_unadjusted AS storage_heater_previous_period_kwh_unadjusted, - storage_heater.tariff_has_changed AS storage_heater_tariff_has_changed, - COALESCE(electricity.comparison_report_id, gas.comparison_report_id, storage_heater.comparison_report_id) AS comparison_report_id, - (electricity.pupils_changed OR gas.pupils_changed OR storage_heater.pupils_changed) AS pupils_changed, - (electricity.floor_area_changed OR gas.floor_area_changed OR storage_heater.floor_area_changed) AS floor_area_changed - FROM (((((latest_runs - JOIN additional ON ((latest_runs.id = additional.alert_generation_run_id))) - LEFT JOIN benchmark ON ((latest_runs.id = benchmark.alert_generation_run_id))) - LEFT JOIN electricity ON ((latest_runs.id = electricity.alert_generation_run_id))) - LEFT JOIN gas ON (((latest_runs.id = gas.alert_generation_run_id) AND ((electricity.comparison_report_id IS NULL) OR (electricity.comparison_report_id = gas.comparison_report_id))))) - LEFT JOIN storage_heater ON (((latest_runs.id = storage_heater.alert_generation_run_id) AND ((gas.comparison_report_id IS NULL) OR (gas.comparison_report_id = storage_heater.comparison_report_id)) AND ((electricity.comparison_report_id IS NULL) OR (electricity.comparison_report_id = storage_heater.comparison_report_id))))); - SQL create_view "electricity_targets", sql_definition: <<-SQL SELECT latest_runs.id, data.alert_generation_run_id, diff --git a/lib/tasks/deployment/20241011123520_add_half_hourly_labelling_to_edf.rake b/lib/tasks/deployment/20241011123520_add_half_hourly_labelling_to_edf.rake new file mode 100644 index 0000000000..720dfa06c2 --- /dev/null +++ b/lib/tasks/deployment/20241011123520_add_half_hourly_labelling_to_edf.rake @@ -0,0 +1,13 @@ +namespace :after_party do + desc 'Deployment task: add_half_hourly_labelling_to_edf' + task add_half_hourly_labelling_to_edf: :environment do + puts "Running deploy task 'add_half_hourly_labelling_to_edf'" + + AmrDataFeedConfig.where(identifier: :edf).update_all(half_hourly_labelling: :start) + + # Update task as completed. If you remove the line below, the task will + # run with every deploy (or every time you call after_party:run). + AfterParty::TaskRecord + .create version: AfterParty::TaskRecorder.new(__FILE__).timestamp + end +end diff --git a/spec/factories/amr_data_feed_config_factory.rb b/spec/factories/amr_data_feed_config_factory.rb index b2d9fb1f07..fb6c41033d 100644 --- a/spec/factories/amr_data_feed_config_factory.rb +++ b/spec/factories/amr_data_feed_config_factory.rb @@ -1,20 +1,45 @@ FactoryBot.define do factory :amr_data_feed_config do source_type { :email } - date_format { '%b %e %Y %I:%M%p' } - sequence(:description) { |n| "Email data feed-#{n}" } - mpan_mprn_field { 'M1_Code1' } - reading_date_field { 'Date' } - reading_time_field { 'Time' } + sequence(:identifier) { |n| "data-config-#{n}" } + sequence(:description) { |n| "Data config #{n}" } number_of_header_rows { 1 } - provider_id_field { 'ID' } - msn_field { 'M1_Code2' } - total_field { 'Total Units' } - meter_description_field { 'Location' } - postcode_field { 'PostCode' } - units_field { 'Units' } - header_example { 'ID,Date,Location,Type,PostCode,Units,Total Units,[00:30],[01:00],[01:30],[02:00],[02:30],[03:00],[03:30],[04:00],[04:30],[05:00],[05:30],[06:00],[06:30],[07:00],[07:30],[08:00],[08:30],[09:00],[09:30],[10:00],[10:30],[11:00],[11:30],[12:00],[12:30],[13:00],[13:30],[14:00],[14:30],[15:00],[15:30],[16:00],[16:30],[17:00],[17:30],[18:00],[18:30],[19:00],[19:30],[20:00],[20:30],[21:00],[21:30],[22:00],[22:30],[23:00],[23:30],[24:00],M1_Code1,M1_Code2' } - reading_fields { '[00:30],[01:00],[01:30],[02:00],[02:30],[03:00],[03:30],[04:00],[04:30],[05:00],[05:30],[06:00],[06:30],[07:00],[07:30],[08:00],[08:30],[09:00],[09:30],[10:00],[10:30],[11:00],[11:30],[12:00],[12:30],[13:00],[13:30],[14:00],[14:30],[15:00],[15:30],[16:00],[16:30],[17:00],[17:30],[18:00],[18:30],[19:00],[19:30],[20:00],[20:30],[21:00],[21:30],[22:00],[22:30],[23:00],[23:30],[24:00]'.split(',') } - sequence(:identifier) { |n| "bucket-#{n}" } + mpan_mprn_field { 'MPRN' } + reading_date_field { 'Date' } + date_format { '%Y-%m-%d' } + header_example do + 'Name,MPAN,Date,00:00,00:30,01:00,01:30,02:00,02:30,03:00,03:30,04:00,04:30,05:00,05:30,06:00,06:30,07:00,07:30,08:00,08:30,09:00,09:30,10:00,10:30,11:00,11:30,12:00,12:30,13:00,13:30,14:00, +14:30,15:00,15:30,16:00,16:30,17:00,17:30,18:00,18:30,19:00,19:30,20:00,20:30,21:00,21:30,22:00,22:30,23:00,23:30' + end + reading_fields do + '00:00,00:30,01:00,01:30,02:00,02:30,03:00,03:30,04:00,04:30,05:00,05:30,06:00,06:30,07:00,07:30,08:00,08:30,09:00,09:30,10:00,10:30,11:00,11:30,12:00,12:30,13:00,13:30,14:00, +14:30,15:00,15:30,16:00,16:30,17:00,17:30,18:00,18:30,19:00,19:30,20:00,20:30,21:00,21:30,22:00,22:30,23:00,23:30'.split(',') + end + + trait :with_row_per_reading do + row_per_reading { true } + header_example { 'MPRN,Date,KWH' } + reading_fields { ['KWH'] } + end + + trait :with_positional_index do + with_row_per_reading + positional_index { true } + period_field { 'Period' } + header_example { 'MPRN,Date,Period,KWH' } + end + + trait :with_reading_time_field do + with_row_per_reading + positional_index { true } + reading_time_field { 'Time' } + header_example { 'MPRN,Date,Time,KWH' } + end + + trait :with_serial_number_lookup do + mpan_mprn_field { '' } + msn_field { 'MSN' } + lookup_by_serial_number { true } + end end end diff --git a/spec/models/amr_data_feed_config_spec.rb b/spec/models/amr_data_feed_config_spec.rb index f3ec85dfc4..7751551d7d 100644 --- a/spec/models/amr_data_feed_config_spec.rb +++ b/spec/models/amr_data_feed_config_spec.rb @@ -1,30 +1,40 @@ require 'rails_helper' describe AmrDataFeedReading do - let(:reading_fields) { '[00:30],[01:00],[01:30],[02:00],[02:30],[03:00],[03:30],[04:00],[04:30],[05:00],[05:30],[06:00],[06:30],[07:00],[07:30],[08:00],[08:30],[09:00],[09:30],[10:00],[10:30],[11:00],[11:30],[12:00],[12:30],[13:00],[13:30],[14:00],[14:30],[15:00],[15:30],[16:00],[16:30],[17:00],[17:30],[18:00],[18:30],[19:00],[19:30],[20:00],[20:30],[21:00],[21:30],[22:00],[22:30],[23:00],[23:30],[24:00]'.split(',') } - let(:header_example) { 'ID,Date,Location,Type,PostCode,Units,Total Units,[00:30],[01:00],[01:30],[02:00],[02:30],[03:00],[03:30],[04:00],[04:30],[05:00],[05:30],[06:00],[06:30],[07:00],[07:30],[08:00],[08:30],[09:00],[09:30],[10:00],[10:30],[11:00],[11:30],[12:00],[12:30],[13:00],[13:30],[14:00],[14:30],[15:00],[15:30],[16:00],[16:30],[17:00],[17:30],[18:00],[18:30],[19:00],[19:30],[20:00],[20:30],[21:00],[21:30],[22:00],[22:30],[23:00],[23:30],[24:00],M1_Code1,M1_Code2'} - - let(:amr_data_feed_config) do - AmrDataFeedConfig.new( - description: 'Banes', - identifier: 'banes', - source_type: :sftp, - date_format: '%b %e %Y %I:%M%p', - mpan_mprn_field: 'M1_Code1', - reading_date_field: 'Date', - reading_fields: reading_fields, - header_example: header_example - ) - end + let(:amr_data_feed_config) { create(:amr_data_feed_config) } + + context 'when validating' do + it { expect(build(:amr_data_feed_config)).to be_valid } - let(:header_example_funny_order) { 'ID,Date,Location,Type,PostCode,Units,Total Units,[00:30],[01:30],[01:00],[02:00],[02:30],[03:00],[03:30],[04:00],[04:30],[05:00],[05:30],[06:00],[06:30],[07:00],[07:30],[08:00],[08:30],[09:00],[09:30],[10:00],[10:30],[11:00],[11:30],[12:00],[12:30],[13:00],[13:30],[14:00],[14:30],[15:00],[15:30],[16:00],[16:30],[17:00],[17:30],[18:00],[18:30],[19:00],[19:30],[20:00],[20:30],[21:00],[21:30],[22:00],[22:30],[23:00],[23:30],[24:00],M1_Code1,M1_Code2'} + context 'with positional index' do + it { expect(build(:amr_data_feed_config, positional_index: true, row_per_reading: false, period_field: nil)).not_to be_valid } + it { expect(build(:amr_data_feed_config, positional_index: true, row_per_reading: false, period_field: 'Period')).not_to be_valid } + it { expect(build(:amr_data_feed_config, positional_index: true, row_per_reading: true, period_field: nil)).not_to be_valid } + it { expect(build(:amr_data_feed_config, :with_positional_index)).to be_valid } + end - it 'correctly identifies the indexes of the reading records' do - expect(amr_data_feed_config.array_of_reading_indexes).to eq (7..7 + 47).to_a + context 'with serial lookup' do + it { expect(build(:amr_data_feed_config, lookup_by_serial_number: true, msn_field: nil)).not_to be_valid } + it { expect(build(:amr_data_feed_config, :with_serial_number_lookup)).to be_valid } + it { expect(build(:amr_data_feed_config, msn_field: 'MSN')).to be_valid } + end end - it 'correctly identifies the indexes of the reading records even when in a funny order' do - amr_data_feed_config.update(header_example: header_example_funny_order) - expect(amr_data_feed_config.array_of_reading_indexes).to eq [7, 9, 8] + (10..10 + 44).to_a + describe '#array_of_reading_indexes' do + it 'correctly identifies the indexes of the reading records' do + expect(amr_data_feed_config.array_of_reading_indexes).to eq (3..3 + 47).to_a + end + + context 'with jumbled order of columns' do + let(:reading_fields) { '[00:30],[01:00],[01:30],[02:00],[02:30],[03:00],[03:30],[04:00],[04:30],[05:00],[05:30],[06:00],[06:30],[07:00],[07:30],[08:00],[08:30],[09:00],[09:30],[10:00],[10:30],[11:00],[11:30],[12:00],[12:30],[13:00],[13:30],[14:00],[14:30],[15:00],[15:30],[16:00],[16:30],[17:00],[17:30],[18:00],[18:30],[19:00],[19:30],[20:00],[20:30],[21:00],[21:30],[22:00],[22:30],[23:00],[23:30],[24:00]'.split(',') } + + let(:header_example) { 'ID,Date,Location,Type,PostCode,Units,Total Units,[00:30],[01:30],[01:00],[02:00],[02:30],[03:00],[03:30],[04:00],[04:30],[05:00],[05:30],[06:00],[06:30],[07:00],[07:30],[08:00],[08:30],[09:00],[09:30],[10:00],[10:30],[11:00],[11:30],[12:00],[12:30],[13:00],[13:30],[14:00],[14:30],[15:00],[15:30],[16:00],[16:30],[17:00],[17:30],[18:00],[18:30],[19:00],[19:30],[20:00],[20:30],[21:00],[21:30],[22:00],[22:30],[23:00],[23:30],[24:00],M1_Code1,M1_Code2'} + + let(:amr_data_feed_config) { create(:amr_data_feed_config, reading_fields: reading_fields, header_example: header_example) } + + it 'correctly identifies the indexes of the reading records even when in a funny order' do + expect(amr_data_feed_config.array_of_reading_indexes).to eq [7, 9, 8] + (10..10 + 44).to_a + end + end end end diff --git a/spec/services/amr/csv_parser_and_upserter_spec.rb b/spec/services/amr/csv_parser_and_upserter_spec.rb index 8b2c00b8b3..f0aa4bdd6f 100644 --- a/spec/services/amr/csv_parser_and_upserter_spec.rb +++ b/spec/services/amr/csv_parser_and_upserter_spec.rb @@ -7,7 +7,22 @@ module Amr include FakeFS::SpecHelpers let(:file_name) { 'example.csv' } - let!(:config) { create(:amr_data_feed_config) } + let!(:config) do + create(:amr_data_feed_config, + description: 'BANES', + mpan_mprn_field: 'M1_Code1', + reading_date_field: 'Date', + date_format: '%b %e %Y %I:%M%p', + number_of_header_rows: 1, + provider_id_field: 'ID', + total_field: 'Total', + meter_description_field: 'Location', + postcode_field: 'PostCode', + units_field: 'Units', + header_example: 'ID,Date,Location,Type,PostCode,Units,Total Units,[00:30],[01:00],[01:30],[02:00],[02:30],[03:00],[03:30],[04:00],[04:30],[05:00],[05:30],[06:00],[06:30],[07:00],[07:30],[08:00],[08:30],[09:00],[09:30],[10:00],[10:30],[11:00],[11:30],[12:00],[12:30],[13:00],[13:30],[14:00],[14:30],[15:00],[15:30],[16:00],[16:30],[17:00],[17:30],[18:00],[18:30],[19:00],[19:30],[20:00],[20:30],[21:00],[21:30],[22:00],[22:30],[23:00],[23:30],[24:00],M1_Code1,M1_Code2', + reading_fields: '[00:30],[01:00],[01:30],[02:00],[02:30],[03:00],[03:30],[04:00],[04:30],[05:00],[05:30],[06:00],[06:30],[07:00],[07:30],[08:00],[08:30],[09:00],[09:30],[10:00],[10:30],[11:00],[11:30],[12:00],[12:30],[13:00],[13:30],[14:00],[14:30],[15:00],[15:30],[16:00],[16:30],[17:00],[17:30],[18:00],[18:30],[19:00],[19:30],[20:00],[20:30],[21:00],[21:30],[22:00],[22:30],[23:00],[23:30],[24:00]'.split(',') + ) + end let!(:frome_config) do create(:amr_data_feed_config, diff --git a/spec/services/amr/single_read_converter_spec.rb b/spec/services/amr/single_read_converter_spec.rb index 2e89e9c5e1..c97f769195 100644 --- a/spec/services/amr/single_read_converter_spec.rb +++ b/spec/services/amr/single_read_converter_spec.rb @@ -2,619 +2,260 @@ module Amr describe SingleReadConverter do - subject(:converter) { described_class.new(amr_data_feed_config, readings) } + subject(:converter) { described_class.new(config, readings) } - let(:amr_data_feed_config) { create(:amr_data_feed_config, row_per_reading: true) } + let(:config) { create(:amr_data_feed_config, row_per_reading: true) } let(:valid_reading_times) do - [ - '00:00', '0:00', - '00:30', '0:30', - '01:00', '1:00', - '01:30', '1:30', - '02:00', '2:00', - '02:30', '2:30', - '03:00', '3:00', - '03:30', '3:30', - '04:00', '4:00', - '04:30', '4:30', - '05:00', '5:00', - '05:30', '5:30', - '06:00', '6:00', - '06:30', '6:30', - '07:00', '7:00', - '07:30', '7:30', - '08:00', '8:00', - '08:30', '8:30', - '09:00', '9:00', - '09:30', '9:30', - '10:00', '10:30', '11:00', '11:30', '12:00', '12:30', '13:00', '13:30', '14:00', '14:30', '15:00', '15:30', '16:00', '16:30', '17:00', '17:30', '18:00', '18:30', '19:00', '19:30', '20:00', '20:30', '21:00', '21:30', '22:00', '22:30', '23:00', '23:30' - ] + 48.times.map do |hh| + TimeOfDay.time_of_day_from_halfhour_index(hh).to_s + end end - context 'normal file format' do - let(:readings) do - [{ :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 00:30:00', :readings => ['14.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 01:00:00', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 01:30:00', :readings => ['15.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 02:00:00', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 02:30:00', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 03:00:00', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 03:30:00', :readings => ['14.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 04:00:00', :readings => ['15.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 04:30:00', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 05:00:00', :readings => ['15.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 05:30:00', :readings => ['15.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 06:00:00', :readings => ['19.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 06:30:00', :readings => ['29.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 07:00:00', :readings => ['29.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 07:30:00', :readings => ['30.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 08:00:00', :readings => ['29.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 08:30:00', :readings => ['34.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 09:00:00', :readings => ['34.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 09:30:00', :readings => ['34.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 10:00:00', :readings => ['33.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 10:30:00', :readings => ['33.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 11:00:00', :readings => ['33.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 11:30:00', :readings => ['33.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 12:00:00', :readings => ['34.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 12:30:00', :readings => ['33.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 13:00:00', :readings => ['34'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 13:30:00', :readings => ['32.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 14:00:00', :readings => ['34.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 14:30:00', :readings => ['35.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 15:00:00', :readings => ['33'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 15:30:00', :readings => ['33.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 16:00:00', :readings => ['32.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 16:30:00', :readings => ['33.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 17:00:00', :readings => ['37.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 17:30:00', :readings => ['38.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 18:00:00', :readings => ['37.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 18:30:00', :readings => ['36.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 19:00:00', :readings => ['32.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 19:30:00', :readings => ['33.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 20:00:00', :readings => ['31.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 20:30:00', :readings => ['27.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 21:00:00', :readings => ['23.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 21:30:00', :readings => ['16.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 22:00:00', :readings => ['16.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 22:30:00', :readings => ['15.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 23:00:00', :readings => ['15.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 23:30:00', :readings => ['15.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 00:00:00', :readings => ['15.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 00:30:00', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 01:00:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 01:30:00', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 02:00:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 02:30:00', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 03:00:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 03:30:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 04:00:00', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 04:30:00', :readings => ['6.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 05:00:00', :readings => ['2.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 05:30:00', :readings => ['3.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 06:00:00', :readings => ['1.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 06:30:00', :readings => ['1.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 07:00:00', :readings => ['2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 07:30:00', :readings => ['3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 08:00:00', :readings => ['2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 08:30:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 09:00:00', :readings => ['1.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 09:30:00', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 10:00:00', :readings => ['1.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 10:30:00', :readings => ['0.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 11:00:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 11:30:00', :readings => ['0.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 12:00:00', :readings => ['1.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 12:30:00', :readings => ['0.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 13:00:00', :readings => ['0.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 13:30:00', :readings => ['0.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 14:00:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 14:30:00', :readings => ['1.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 15:00:00', :readings => ['1.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 15:30:00', :readings => ['2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 16:00:00', :readings => ['2.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 16:30:00', :readings => ['3.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 17:00:00', :readings => ['1.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 17:30:00', :readings => ['0.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 18:00:00', :readings => ['0.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 18:30:00', :readings => ['0.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 19:00:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 19:30:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 20:00:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 20:30:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 21:00:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 21:30:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 22:00:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 22:30:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 23:00:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 23:30:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '28 Aug 2019 00:00:00', :readings => ['1.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 00:30:00', :readings => ['1.4'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 01:00:00', :readings => ['1.3'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 01:30:00', :readings => ['1.4'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 02:00:00', :readings => ['1.3'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 02:30:00', :readings => ['1.4'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 03:00:00', :readings => ['1.3'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 03:30:00', :readings => ['1.3'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 04:00:00', :readings => ['1.4'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 04:30:00', :readings => ['6.5'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 05:00:00', :readings => ['2.3'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 05:30:00', :readings => ['3.2'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 06:00:00', :readings => ['1.8'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 06:30:00', :readings => ['1.6'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 07:00:00', :readings => ['2'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 07:30:00', :readings => ['3'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 08:00:00', :readings => ['2'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 08:30:00', :readings => ['1.3'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 09:00:00', :readings => ['1.7'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 09:30:00', :readings => ['1.4'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 10:00:00', :readings => ['1.1'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 10:30:00', :readings => ['0.9'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 11:00:00', :readings => ['1.2'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 11:30:00', :readings => ['0.9'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 12:00:00', :readings => ['1.7'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 12:30:00', :readings => ['0.8'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 13:00:00', :readings => ['0.8'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 13:30:00', :readings => ['0.8'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 14:00:00', :readings => ['1.2'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 14:30:00', :readings => ['1.1'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 15:00:00', :readings => ['1.7'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 15:30:00', :readings => ['2'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 16:00:00', :readings => ['2.8'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 16:30:00', :readings => ['3.8'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 17:00:00', :readings => ['1.6'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 17:30:00', :readings => ['0.5'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 18:00:00', :readings => ['0.7'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 18:30:00', :readings => ['0.9'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 19:00:00', :readings => ['1.2'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 19:30:00', :readings => ['1.2'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 20:00:00', :readings => ['1.2'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 20:30:00', :readings => ['1.3'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 21:00:00', :readings => ['1.3'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 21:30:00', :readings => ['1.2'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 22:00:00', :readings => ['1.2'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 22:30:00', :readings => ['1.2'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 23:00:00', :readings => ['1.3'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '27 Aug 2019 23:30:00', :readings => ['1.2'], meter_id: 123 }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '28 Aug 2019 00:00:00', :readings => ['1.6'], meter_id: 123 }, - ] - end + def create_hh_readings + 48.times.collect { |i| (i + 1).to_f } + end - let(:output) do - [{ amr_data_feed_config_id: 6, meter_id: nil, reading_date: Date.parse('26 Aug 2019'), mpan_mprn: '1710035168313', readings: [14.4, 15.0, 15.1, 15.0, 15.0, 15.0, 14.9, 15.1, 15.0, 15.1, 15.7, 19.6, 29.9, 29.7, 30.2, 29.6, 34.1, 34.4, 34.7, 33.5, 33.5, 33.4, 33.6, 34.5, 33.7, 34.0, 32.7, 34.2, 35.1, 33.0, 33.4, 32.6, 33.1, 37.6, 38.9, 37.7, 36.7, 32.9, 33.1, 31.1, 27.6, 23.3, 16.7, 16.6, 15.9, 15.6, 15.4, 15.2] }, - { amr_data_feed_config_id: 6, meter_id: nil, reading_date: Date.parse('27 Aug 2019'), mpan_mprn: '1710035168313', readings: [1.4, 1.3, 1.4, 1.3, 1.4, 1.3, 1.3, 1.4, 6.5, 2.3, 3.2, 1.8, 1.6, 2.0, 3.0, 2.0, 1.3, 1.7, 1.4, 1.1, 0.9, 1.2, 0.9, 1.7, 0.8, 0.8, 0.8, 1.2, 1.1, 1.7, 2.0, 2.8, 3.8, 1.6, 0.5, 0.7, 0.9, 1.2, 1.2, 1.2, 1.3, 1.3, 1.2, 1.2, 1.2, 1.3, 1.2, 1.6] }, - { amr_data_feed_config_id: 6, meter_id: 123, reading_date: Date.parse('27 Aug 2019'), mpan_mprn: '1710035168314', readings: [1.4, 1.3, 1.4, 1.3, 1.4, 1.3, 1.3, 1.4, 6.5, 2.3, 3.2, 1.8, 1.6, 2.0, 3.0, 2.0, 1.3, 1.7, 1.4, 1.1, 0.9, 1.2, 0.9, 1.7, 0.8, 0.8, 0.8, 1.2, 1.1, 1.7, 2.0, 2.8, 3.8, 1.6, 0.5, 0.7, 0.9, 1.2, 1.2, 1.2, 1.3, 1.3, 1.2, 1.2, 1.2, 1.3, 1.2, 1.6] },] - end + def create_reading(config, mpan_mprn, reading_date, readings, meter_id: nil) + { + amr_data_feed_config_id: config.id, + meter_id: meter_id, + mpan_mprn: mpan_mprn, + reading_date: reading_date, + readings: readings + } + end - it 'converts a list of single readings per half hour into a day per reading format' do - expect(converter.perform).to eq output - end + def create_reading_for_period(config, mpan_mprn, reading_date, period, readings, meter_id: nil) + create_reading(config, mpan_mprn, reading_date, readings, meter_id: meter_id).merge({ period: period }) + end - context 'with a data with missing readings' do - subject(:converter) { described_class.new(amr_data_feed_config, with_missing_readings) } + def create_reading_for_time(config, mpan_mprn, reading_date, reading_time, readings, meter_id: nil) + create_reading(config, mpan_mprn, reading_date, readings, meter_id: meter_id).merge({ reading_time: reading_time }) + end - let(:with_missing_readings) do - readings << { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168314', :reading_date => '29 Aug 2019 00:30:00', :readings => ['1.6'], meter_id: 123 } + context 'with a reading timestamp column' do + # This is the EnergySparks default. + # Matches the EDF format + # So 26 Aug 2019 00:00 means usage FROM midnight to 00:30am on the 26th August + # So 26 Aug 2019 23:30 means usage FROM 23:30 to midnight on the 26th August + context 'with readings labelled at start of the half hour, so 00:00 is start of the day (%H:%M:%s) and correct config' do + let(:config) { create(:amr_data_feed_config, :with_row_per_reading, half_hourly_labelling: :start, date_format: '%d %b %Y %H:%M') } + + let(:mpan_mprn) { '1710035168313' } + let(:reading_date) { '26 Aug 2019' } + let(:readings) do + 48.times.collect do |hh| + hh_time = TimeOfDay.time_of_day_from_halfhour_index(hh).to_s + create_reading(config, mpan_mprn, "#{reading_date} #{hh_time}:00", [(hh + 1).to_s]) + end end - - it 'drops the row' do - expect(converter.perform).to eq output + let(:expected_output) do + [create_reading(config, mpan_mprn, Date.parse(reading_date), 48.times.collect { |i| (i + 1) })] end - end - end - context 'offset file format' do - let(:readings) do - [{ :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 00:00:00', :readings => ['13.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 00:30:00', :readings => ['14.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 01:00:00', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 01:30:00', :readings => ['15.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 02:00:00', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 02:30:00', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 03:00:00', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 03:30:00', :readings => ['14.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 04:00:00', :readings => ['15.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 04:30:00', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 05:00:00', :readings => ['15.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 05:30:00', :readings => ['15.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 06:00:00', :readings => ['19.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 06:30:00', :readings => ['29.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 07:00:00', :readings => ['29.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 07:30:00', :readings => ['30.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 08:00:00', :readings => ['29.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 08:30:00', :readings => ['34.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 09:00:00', :readings => ['34.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 09:30:00', :readings => ['34.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 10:00:00', :readings => ['33.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 10:30:00', :readings => ['33.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 11:00:00', :readings => ['33.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 11:30:00', :readings => ['33.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 12:00:00', :readings => ['34.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 12:30:00', :readings => ['33.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 13:00:00', :readings => ['34'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 13:30:00', :readings => ['32.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 14:00:00', :readings => ['34.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 14:30:00', :readings => ['35.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 15:00:00', :readings => ['33'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 15:30:00', :readings => ['33.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 16:00:00', :readings => ['32.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 16:30:00', :readings => ['33.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 17:00:00', :readings => ['37.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 17:30:00', :readings => ['38.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 18:00:00', :readings => ['37.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 18:30:00', :readings => ['36.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 19:00:00', :readings => ['32.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 19:30:00', :readings => ['33.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 20:00:00', :readings => ['31.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 20:30:00', :readings => ['27.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 21:00:00', :readings => ['23.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 21:30:00', :readings => ['16.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 22:00:00', :readings => ['16.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 22:30:00', :readings => ['15.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 23:00:00', :readings => ['15.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '26 Aug 2019 23:30:00', :readings => ['15.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 00:00:00', :readings => ['15.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 00:30:00', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 01:00:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 01:30:00', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 02:00:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 02:30:00', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 03:00:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 03:30:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 04:00:00', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 04:30:00', :readings => ['6.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 05:00:00', :readings => ['2.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 05:30:00', :readings => ['3.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 06:00:00', :readings => ['1.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 06:30:00', :readings => ['1.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 07:00:00', :readings => ['2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 07:30:00', :readings => ['3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 08:00:00', :readings => ['2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 08:30:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 09:00:00', :readings => ['1.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 09:30:00', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 10:00:00', :readings => ['1.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 10:30:00', :readings => ['0.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 11:00:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 11:30:00', :readings => ['0.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 12:00:00', :readings => ['1.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 12:30:00', :readings => ['0.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 13:00:00', :readings => ['0.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 13:30:00', :readings => ['0.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 14:00:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 14:30:00', :readings => ['1.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 15:00:00', :readings => ['1.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 15:30:00', :readings => ['2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 16:00:00', :readings => ['2.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 16:30:00', :readings => ['3.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 17:00:00', :readings => ['1.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 17:30:00', :readings => ['0.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 18:00:00', :readings => ['0.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 18:30:00', :readings => ['0.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 19:00:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 19:30:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 20:00:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 20:30:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 21:00:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 21:30:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 22:00:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 22:30:00', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 23:00:00', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => '27 Aug 2019 23:30:00', :readings => ['1.2'] }, - ] + it 'converts correctly' do + expect(converter.perform).to eq expected_output + end end - let(:output) do - [ - { amr_data_feed_config_id: 6, meter_id: nil, reading_date: Date.parse('26 Aug 2019'), mpan_mprn: '1710035168313', readings: [14.4, 15.0, 15.1, 15.0, 15.0, 15.0, 14.9, 15.1, 15.0, 15.1, 15.7, 19.6, 29.9, 29.7, 30.2, 29.6, 34.1, 34.4, 34.7, 33.5, 33.5, 33.4, 33.6, 34.5, 33.7, 34.0, 32.7, 34.2, 35.1, 33.0, 33.4, 32.6, 33.1, 37.6, 38.9, 37.7, 36.7, 32.9, 33.1, 31.1, 27.6, 23.3, 16.7, 16.6, 15.9, 15.6, 15.4, 15.2] }, - { amr_data_feed_config_id: 6, meter_id: nil, reading_date: Date.parse('27 Aug 2019'), mpan_mprn: '1710035168313', readings: [1.4, 1.3, 1.4, 1.3, 1.4, 1.3, 1.3, 1.4, 6.5, 2.3, 3.2, 1.8, 1.6, 2.0, 3.0, 2.0, 1.3, 1.7, 1.4, 1.1, 0.9, 1.2, 0.9, 1.7, 0.8, 0.8, 0.8, 1.2, 1.1, 1.7, 2.0, 2.8, 3.8, 1.6, 0.5, 0.7, 0.9, 1.2, 1.2, 1.2, 1.3, 1.3, 1.2, 1.2, 1.2, 1.3, 1.2, nil] }, - ] - end + # Opposite to our default assumption + # So 26 Aug 2019 23:30 means usage UP TO 23:30 on 26th August + # So 27 Aug 2019 00:00 means usage FROM 23:30 on 26th August to 00:00 + context 'with readings labelled at end of the half hour so 00:00 is end of the day' do + let(:mpan_mprn) { '1710035168313' } + + context 'with date times formatted as %d %b %Y %H:%M' do + let(:reading_date) { Date.parse('26 Aug 2019') } + let(:meter_id) { nil } + # This is testing with timestamps from 26 Aug 2019 00:30 to 26 Aug 23:30 plus 27 Aug 00:00 + let(:readings) do + 48.times.collect do |hh| + date = hh < 47 ? reading_date.strftime('%d %b %Y') : (reading_date + 1).strftime('%d %b %Y') + date_time = hh < 47 ? "#{date} #{TimeOfDay.time_of_day_from_halfhour_index(hh + 1)}" : "#{date} 00:00" + create_reading(config, mpan_mprn, date_time, [(hh + 1).to_s], meter_id: meter_id) + end + end + + let(:expected_output) do + [create_reading(config, mpan_mprn, reading_date, create_hh_readings)] + end + + it 'converts correctly' do + expect(converter.perform).to eq expected_output + end + + context 'when readings are matched to meter' do + let(:meter_id) { 1 } + + let(:expected_output) do + [create_reading(config, mpan_mprn, reading_date, create_hh_readings, meter_id: meter_id)] + end + + it 'preserves the ids' do + expect(converter.perform).to eq expected_output + end + end + end - it 'converts a list of single readings per half hour into a day per reading format' do - expect(converter.perform).to eq output + # TODO: this fails locally, but not on github. We end up with 26 Aug 2019 having 49 values with 2 nils, plus + # 27th August having 48 nil values and a single value (48) a position 1 in the array + context 'with date times formatted as ISO 8601 (as produced by xlsx to csv conversion)' do + let(:reading_date) { Time.zone.parse('26 Aug 2019') } + + # this is testing with timestamps from 2019-08-26T00:30:00Z to 2019-08-27T00:00:00Z. + let(:readings) do + 48.times.collect do |i| + { :amr_data_feed_config_id => config.id, :mpan_mprn => mpan_mprn, :reading_date => (reading_date + ((i + 1) * 30).minutes).iso8601, :readings => [(i + 1).to_s] } + end + end + + let(:expected_output) do + [create_reading(config, mpan_mprn, reading_date.to_date, create_hh_readings)] + end + + it 'converts correctly' do + expect(converter.perform).to eq expected_output + end + end end end - context 'With reading dates in ISO 8601 format (produced by xlsx to csv conversion)' do - let(:reading_date) { Time.zone.parse('26 Aug 2019') } - let(:readings) { 48.times.collect {|i| { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', :reading_date => (reading_date + ((i + 1) * 30).minutes).iso8601, :readings => [(i + 1).to_s] } } } - let(:output) { [{ amr_data_feed_config_id: 6, meter_id: nil, reading_date: reading_date.to_date, mpan_mprn: '1710035168313', readings: 48.times.collect {|i| (i + 1) } }] } + context 'with separate date and time columns' do + let(:config) { create(:amr_data_feed_config, :with_reading_time_field) } - it 'converts a list of single readings per half hour into a day per reading format' do - expect(converter.perform).to eq output - end - end + let(:mpan_mprn) { '1710035168313' } + let(:reading_date) { '26 Aug 2019' } - context 'split date and time column file format' do - let(:amr_data_feed_config) { create(:amr_data_feed_config, row_per_reading: true, positional_index: true) } - let(:readings) do - [ - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '00:00', :reading_date => '26/08/2019', :readings => ['14.4'] }, # 1 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '00:30', :reading_date => '26/08/2019', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '01:00', :reading_date => '26/08/2019', :readings => ['15.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '01:30', :reading_date => '26/08/2019', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '02:00', :reading_date => '26/08/2019', :readings => ['15'] }, # 5 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '02:30', :reading_date => '26/08/2019', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '03:00', :reading_date => '26/08/2019', :readings => ['14.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '03:30', :reading_date => '26/08/2019', :readings => ['15.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '04:00', :reading_date => '26/08/2019', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '04:30', :reading_date => '26/08/2019', :readings => ['15.1'] }, # 10 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '05:00', :reading_date => '26/08/2019', :readings => ['15.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '05:30', :reading_date => '26/08/2019', :readings => ['19.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '06:00', :reading_date => '26/08/2019', :readings => ['29.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '06:30', :reading_date => '26/08/2019', :readings => ['29.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '07:00', :reading_date => '26/08/2019', :readings => ['30.2'] }, # 15 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '07:30', :reading_date => '26/08/2019', :readings => ['29.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '08:00', :reading_date => '26/08/2019', :readings => ['34.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '08:30', :reading_date => '26/08/2019', :readings => ['34.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '09:00', :reading_date => '26/08/2019', :readings => ['34.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '09:30', :reading_date => '26/08/2019', :readings => ['33.5'] }, # 20 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '10:00', :reading_date => '26/08/2019', :readings => ['33.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '10:30', :reading_date => '26/08/2019', :readings => ['33.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '11:00', :reading_date => '26/08/2019', :readings => ['33.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '11:30', :reading_date => '26/08/2019', :readings => ['34.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '12:00', :reading_date => '26/08/2019', :readings => ['33.7'] }, # 25 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '12:30', :reading_date => '26/08/2019', :readings => ['34'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '13:00', :reading_date => '26/08/2019', :readings => ['32.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '13:30', :reading_date => '26/08/2019', :readings => ['34.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '14:00', :reading_date => '26/08/2019', :readings => ['35.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '14:30', :reading_date => '26/08/2019', :readings => ['33'] }, # 30 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '15:00', :reading_date => '26/08/2019', :readings => ['33.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '15:30', :reading_date => '26/08/2019', :readings => ['32.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '16:00', :reading_date => '26/08/2019', :readings => ['33.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '16:30', :reading_date => '26/08/2019', :readings => ['37.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '17:00', :reading_date => '26/08/2019', :readings => ['38.9'] }, # 35 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '17:30', :reading_date => '26/08/2019', :readings => ['37.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '18:00', :reading_date => '26/08/2019', :readings => ['36.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '18:30', :reading_date => '26/08/2019', :readings => ['32.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '19:00', :reading_date => '26/08/2019', :readings => ['33.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '19:30', :reading_date => '26/08/2019', :readings => ['31.1'] }, # 40 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '20:00', :reading_date => '26/08/2019', :readings => ['27.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '20:30', :reading_date => '26/08/2019', :readings => ['23.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '21:00', :reading_date => '26/08/2019', :readings => ['16.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '21:30', :reading_date => '26/08/2019', :readings => ['16.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '22:00', :reading_date => '26/08/2019', :readings => ['15.9'] }, # 45 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '22:30', :reading_date => '26/08/2019', :readings => ['15.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '23:00', :reading_date => '26/08/2019', :readings => ['15.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '23:30', :reading_date => '26/08/2019', :readings => ['48.0'] }, # 48 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '0', :reading_date => '27/08/2019', :readings => ['15.2'] }, # 1 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '30', :reading_date => '27/08/2019', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '100', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '130', :reading_date => '27/08/2019', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '200', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '230', :reading_date => '27/08/2019', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '300', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '330', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '400', :reading_date => '27/08/2019', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '430', :reading_date => '27/08/2019', :readings => ['6.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '500', :reading_date => '27/08/2019', :readings => ['2.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '530', :reading_date => '27/08/2019', :readings => ['3.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '600', :reading_date => '27/08/2019', :readings => ['1.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '630', :reading_date => '27/08/2019', :readings => ['1.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '700', :reading_date => '27/08/2019', :readings => ['2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '730', :reading_date => '27/08/2019', :readings => ['3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '800', :reading_date => '27/08/2019', :readings => ['2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '830', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '900', :reading_date => '27/08/2019', :readings => ['1.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '930', :reading_date => '27/08/2019', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1000', :reading_date => '27/08/2019', :readings => ['1.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1030', :reading_date => '27/08/2019', :readings => ['0.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1100', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1130', :reading_date => '27/08/2019', :readings => ['0.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1200', :reading_date => '27/08/2019', :readings => ['1.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1230', :reading_date => '27/08/2019', :readings => ['0.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1300', :reading_date => '27/08/2019', :readings => ['0.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1330', :reading_date => '27/08/2019', :readings => ['0.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1400', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1430', :reading_date => '27/08/2019', :readings => ['1.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1500', :reading_date => '27/08/2019', :readings => ['1.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1530', :reading_date => '27/08/2019', :readings => ['2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1600', :reading_date => '27/08/2019', :readings => ['2.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1630', :reading_date => '27/08/2019', :readings => ['3.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1700', :reading_date => '27/08/2019', :readings => ['1.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1730', :reading_date => '27/08/2019', :readings => ['0.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1800', :reading_date => '27/08/2019', :readings => ['0.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1830', :reading_date => '27/08/2019', :readings => ['0.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1900', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '1930', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '2000', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '2030', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '2100', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '2130', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '2200', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '2230', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '2300', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', reading_time: '2330', :reading_date => '27/08/2019', :readings => ['99.0'] } - ] + let(:expected_output) do + [create_reading(config, mpan_mprn, Date.parse(reading_date), 48.times.collect { |i| (i + 1) })] end - let(:output) do - [ - { amr_data_feed_config_id: 6, meter_id: nil, reading_date: Date.parse('26 Aug 2019'), mpan_mprn: '1710035168313', readings: [14.4, 15.0, 15.1, 15.0, 15.0, 15.0, 14.9, 15.1, 15.0, 15.1, 15.7, 19.6, 29.9, 29.7, 30.2, 29.6, 34.1, 34.4, 34.7, 33.5, 33.5, 33.4, 33.6, 34.5, 33.7, 34.0, 32.7, 34.2, 35.1, 33.0, 33.4, 32.6, 33.1, 37.6, 38.9, 37.7, 36.7, 32.9, 33.1, 31.1, 27.6, 23.3, 16.7, 16.6, 15.9, 15.6, 15.4, 48.0] }, - { amr_data_feed_config_id: 6, meter_id: nil, reading_date: Date.parse('27 Aug 2019'), mpan_mprn: '1710035168313', readings: [15.2, 1.4, 1.3, 1.4, 1.3, 1.4, 1.3, 1.3, 1.4, 6.5, 2.3, 3.2, 1.8, 1.6, 2.0, 3.0, 2.0, 1.3, 1.7, 1.4, 1.1, 0.9, 1.2, 0.9, 1.7, 0.8, 0.8, 0.8, 1.2, 1.1, 1.7, 2.0, 2.8, 3.8, 1.6, 0.5, 0.7, 0.9, 1.2, 1.2, 1.2, 1.3, 1.3, 1.2, 1.2, 1.2, 1.3, 99.0] }, - ] - end + context 'with times formatted as %H:%M' do + let(:readings) do + valid_reading_times.each.with_index(1).map do |hh_time, index| + create_reading_for_time(config, mpan_mprn, reading_date, hh_time, [index.to_s]) + end + end - it 'converts a list of single readings per half hour into a day per reading format' do - results = converter.perform - expect(results).to eq output + it 'converts correctly' do + expect(converter.perform).to eq expected_output + end end - it 'handles files with multiple mpans' do - # create test data that consists of 2 days readings for 2 different meters - two_meters_worth_of_readings = readings + readings.map {|r| { amr_data_feed_config_id: 6, mpan_mprn: '123456789012', reading_date: r[:reading_date], reading_time: r[:reading_time], readings: r[:readings] } } + context 'with times formatted as %H%M' do + let(:readings) do + valid_reading_times.each.with_index(1).map do |hh_time, index| + create_reading_for_time(config, mpan_mprn, reading_date, hh_time.sub(':', ''), [index.to_s]) + end + end - results = described_class.new(amr_data_feed_config, two_meters_worth_of_readings).perform + it 'converts correctly' do + expect(converter.perform).to eq expected_output + end - # create expected output: 2 x 2 days readings for 2 meters - expected_results = output + output.map {|r| { amr_data_feed_config_id: 6, meter_id: nil, mpan_mprn: '123456789012', reading_date: r[:reading_date], readings: r[:readings] } } + it 'handles files with multiple mpans' do + # create test data that consists of 2 days readings for 2 different meters + two_meters_worth_of_readings = readings + readings.map {|r| { amr_data_feed_config_id: 6, mpan_mprn: '123456789012', reading_date: r[:reading_date], reading_time: r[:reading_time], readings: r[:readings] } } - expect(results).to eq expected_results + results = described_class.new(config, two_meters_worth_of_readings).perform + + # create expected output: 2 x 2 days readings for 2 meters + expected_results = expected_output + expected_output.map {|r| { amr_data_feed_config_id: 6, meter_id: nil, mpan_mprn: '123456789012', reading_date: r[:reading_date], readings: r[:readings] } } + + expect(results).to eq expected_results + end + end + + context 'with times formatted without padding (0, 30, 100, 130..2330)' do + let(:readings) do + valid_reading_times.each.with_index(1).map do |hh_time, index| + time = index == 1 ? '0' : hh_time.sub(':', '').gsub(/^0+/, '') + create_reading_for_time(config, mpan_mprn, reading_date, time, [index.to_s]) + end + end + + it 'converts correctly' do + expect(converter.perform).to eq expected_output + end end end - context 'indexed file format' do - let(:amr_data_feed_config) { create(:amr_data_feed_config, row_per_reading: true, positional_index: true) } + context 'with numbered half-hourly periods (positional_index: true)' do + let(:config) { create(:amr_data_feed_config, :with_positional_index) } + + let(:mpan_mprn) { '1710035168313' } + let(:reading_date) { '26 Aug 2019' } let(:readings) do - [{ :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '1', :reading_date => '26/08/2019', :readings => ['14.4'] }, # 1 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '2', :reading_date => '26/08/2019', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '3', :reading_date => '26/08/2019', :readings => ['15.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '4', :reading_date => '26/08/2019', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '5', :reading_date => '26/08/2019', :readings => ['15'] }, # 5 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '6', :reading_date => '26/08/2019', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '7', :reading_date => '26/08/2019', :readings => ['14.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '8', :reading_date => '26/08/2019', :readings => ['15.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '9', :reading_date => '26/08/2019', :readings => ['15'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '10', :reading_date => '26/08/2019', :readings => ['15.1'] }, # 10 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '11', :reading_date => '26/08/2019', :readings => ['15.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '12', :reading_date => '26/08/2019', :readings => ['19.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '13', :reading_date => '26/08/2019', :readings => ['29.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '14', :reading_date => '26/08/2019', :readings => ['29.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '15', :reading_date => '26/08/2019', :readings => ['30.2'] }, # 15 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '16', :reading_date => '26/08/2019', :readings => ['29.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '17', :reading_date => '26/08/2019', :readings => ['34.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '18', :reading_date => '26/08/2019', :readings => ['34.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '19', :reading_date => '26/08/2019', :readings => ['34.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '20', :reading_date => '26/08/2019', :readings => ['33.5'] }, # 20 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '21', :reading_date => '26/08/2019', :readings => ['33.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '22', :reading_date => '26/08/2019', :readings => ['33.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '23', :reading_date => '26/08/2019', :readings => ['33.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '24', :reading_date => '26/08/2019', :readings => ['34.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '25', :reading_date => '26/08/2019', :readings => ['33.7'] }, # 25 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '26', :reading_date => '26/08/2019', :readings => ['34'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '27', :reading_date => '26/08/2019', :readings => ['32.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '28', :reading_date => '26/08/2019', :readings => ['34.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '29', :reading_date => '26/08/2019', :readings => ['35.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '30', :reading_date => '26/08/2019', :readings => ['33'] }, # 30 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '31', :reading_date => '26/08/2019', :readings => ['33.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '32', :reading_date => '26/08/2019', :readings => ['32.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '33', :reading_date => '26/08/2019', :readings => ['33.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '34', :reading_date => '26/08/2019', :readings => ['37.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '35', :reading_date => '26/08/2019', :readings => ['38.9'] }, # 35 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '36', :reading_date => '26/08/2019', :readings => ['37.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '37', :reading_date => '26/08/2019', :readings => ['36.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '38', :reading_date => '26/08/2019', :readings => ['32.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '39', :reading_date => '26/08/2019', :readings => ['33.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '40', :reading_date => '26/08/2019', :readings => ['31.1'] }, # 40 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '41', :reading_date => '26/08/2019', :readings => ['27.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '42', :reading_date => '26/08/2019', :readings => ['23.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '43', :reading_date => '26/08/2019', :readings => ['16.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '44', :reading_date => '26/08/2019', :readings => ['16.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '45', :reading_date => '26/08/2019', :readings => ['15.9'] }, # 45 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '46', :reading_date => '26/08/2019', :readings => ['15.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '47', :reading_date => '26/08/2019', :readings => ['15.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '48', :reading_date => '26/08/2019', :readings => ['48.0'] }, # 48 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '1', :reading_date => '27/08/2019', :readings => ['15.2'] }, # 1 - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '2', :reading_date => '27/08/2019', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '3', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '4', :reading_date => '27/08/2019', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '5', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '6', :reading_date => '27/08/2019', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '7', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '8', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '9', :reading_date => '27/08/2019', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '10', :reading_date => '27/08/2019', :readings => ['6.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '11', :reading_date => '27/08/2019', :readings => ['2.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '12', :reading_date => '27/08/2019', :readings => ['3.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '13', :reading_date => '27/08/2019', :readings => ['1.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '14', :reading_date => '27/08/2019', :readings => ['1.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '15', :reading_date => '27/08/2019', :readings => ['2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '16', :reading_date => '27/08/2019', :readings => ['3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '17', :reading_date => '27/08/2019', :readings => ['2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '18', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '19', :reading_date => '27/08/2019', :readings => ['1.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '20', :reading_date => '27/08/2019', :readings => ['1.4'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '21', :reading_date => '27/08/2019', :readings => ['1.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '22', :reading_date => '27/08/2019', :readings => ['0.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '23', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '24', :reading_date => '27/08/2019', :readings => ['0.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '25', :reading_date => '27/08/2019', :readings => ['1.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '26', :reading_date => '27/08/2019', :readings => ['0.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '27', :reading_date => '27/08/2019', :readings => ['0.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '28', :reading_date => '27/08/2019', :readings => ['0.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '29', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '30', :reading_date => '27/08/2019', :readings => ['1.1'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '31', :reading_date => '27/08/2019', :readings => ['1.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '32', :reading_date => '27/08/2019', :readings => ['2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '33', :reading_date => '27/08/2019', :readings => ['2.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '34', :reading_date => '27/08/2019', :readings => ['3.8'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '35', :reading_date => '27/08/2019', :readings => ['1.6'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '36', :reading_date => '27/08/2019', :readings => ['0.5'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '37', :reading_date => '27/08/2019', :readings => ['0.7'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '38', :reading_date => '27/08/2019', :readings => ['0.9'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '39', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '40', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '41', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '42', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '43', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '44', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '45', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '46', :reading_date => '27/08/2019', :readings => ['1.2'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '47', :reading_date => '27/08/2019', :readings => ['1.3'] }, - { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: '48', :reading_date => '27/08/2019', :readings => ['99.0'] }, # 48 - ] + 48.times.collect { |hh| create_reading_for_period(config, mpan_mprn, reading_date, (hh + 1).to_s, [(hh + 1).to_s]) } end - let(:output) do - [ - { amr_data_feed_config_id: 6, meter_id: nil, reading_date: Date.parse('26 Aug 2019'), mpan_mprn: '1710035168313', readings: [14.4, 15.0, 15.1, 15.0, 15.0, 15.0, 14.9, 15.1, 15.0, 15.1, 15.7, 19.6, 29.9, 29.7, 30.2, 29.6, 34.1, 34.4, 34.7, 33.5, 33.5, 33.4, 33.6, 34.5, 33.7, 34.0, 32.7, 34.2, 35.1, 33.0, 33.4, 32.6, 33.1, 37.6, 38.9, 37.7, 36.7, 32.9, 33.1, 31.1, 27.6, 23.3, 16.7, 16.6, 15.9, 15.6, 15.4, 48.0] }, - { amr_data_feed_config_id: 6, meter_id: nil, reading_date: Date.parse('27 Aug 2019'), mpan_mprn: '1710035168313', readings: [15.2, 1.4, 1.3, 1.4, 1.3, 1.4, 1.3, 1.3, 1.4, 6.5, 2.3, 3.2, 1.8, 1.6, 2.0, 3.0, 2.0, 1.3, 1.7, 1.4, 1.1, 0.9, 1.2, 0.9, 1.7, 0.8, 0.8, 0.8, 1.2, 1.1, 1.7, 2.0, 2.8, 3.8, 1.6, 0.5, 0.7, 0.9, 1.2, 1.2, 1.2, 1.3, 1.3, 1.2, 1.2, 1.2, 1.3, 99.0] }, - ] + let(:expected_output) do + [create_reading(config, mpan_mprn, Date.parse(reading_date), create_hh_readings)] end - it 'converts a list of single readings per half hour into a day per reading format' do - results = converter.perform - expect(results).to eq output + it 'converts correctly' do + expect(converter.perform).to eq expected_output end it 'handles files with multiple mpans' do # create test data that consists of 2 days readings for 2 different meters two_meters_worth_of_readings = readings + readings.map {|r| { amr_data_feed_config_id: 6, mpan_mprn: '123456789012', reading_date: r[:reading_date], period: r[:period], readings: r[:readings] } } - results = described_class.new(amr_data_feed_config, two_meters_worth_of_readings).perform + results = described_class.new(config, two_meters_worth_of_readings).perform # create expected output: 2 x 2 days readings for 2 meters - expected_results = output + output.map {|r| { amr_data_feed_config_id: 6, meter_id: nil, mpan_mprn: '123456789012', reading_date: r[:reading_date], readings: r[:readings] } } + expected_results = expected_output + expected_output.map {|r| { amr_data_feed_config_id: 6, meter_id: nil, mpan_mprn: '123456789012', reading_date: r[:reading_date], readings: r[:readings] } } expect(results).to eq expected_results end end - context 'missing mpan_mprn' do - let(:readings) { [{ :amr_data_feed_config_id => 6, :mpan_mprn => nil, reading_date: Date.parse('27 Aug 2019').to_s, readings: [15.2, 1.4, 1.3, 1.4, 1.3, 1.4, 1.3, 1.3, 1.4, 6.5, 2.3, 3.2, 1.8, 1.6, 2.0, 3.0, 2.0, 1.3, 1.7, 1.4, 1.1, 0.9, 1.2, 0.9, 1.7, 0.8, 0.8, 0.8, 1.2, 1.1, 1.7, 2.0, 2.8, 3.8, 1.6, 0.5, 0.7, 0.9, 1.2, 1.2, 1.2, 1.3, 1.3, 1.2, 1.2, 1.2, 1.3, 99.0] }] } + context 'when data is invalid' do + context 'with missing mpan_mprn' do + let(:readings) { [create_reading(config, nil, '27 Aug 2019', create_hh_readings)] } - it 'ignores row' do - results = converter.perform - expect(results).to be_empty + it 'ignores row' do + expect(converter.perform).to be_empty + end end - end - context 'missing date' do - let(:readings) { [{ :amr_data_feed_config_id => 6, :mpan_mprn => '12345678', reading_date: nil, readings: [15.2, 1.4, 1.3, 1.4, 1.3, 1.4, 1.3, 1.3, 1.4, 6.5, 2.3, 3.2, 1.8, 1.6, 2.0, 3.0, 2.0, 1.3, 1.7, 1.4, 1.1, 0.9, 1.2, 0.9, 1.7, 0.8, 0.8, 0.8, 1.2, 1.1, 1.7, 2.0, 2.8, 3.8, 1.6, 0.5, 0.7, 0.9, 1.2, 1.2, 1.2, 1.3, 1.3, 1.2, 1.2, 1.2, 1.3, 99.0] }] } + context 'with missing date' do + let(:readings) { [create_reading(config, '12345678', nil, create_hh_readings)] } - it 'ignores row' do - results = converter.perform - expect(results).to be_empty + it 'ignores row' do + expect(converter.perform).to be_empty + end end - end - context 'dodgy data' do - let(:readings) { [{ :amr_data_feed_config_id => 6, :mpan_mprn => 'Primary school', :reading_date => '123456789012', :readings => ['01/01/2019'] }] } + context 'with badly parsed values' do + let(:readings) { [create_reading(config, 'Primary school', '123456789012', ['01/01/2019'])] } + + it 'raises an error' do + expect { converter.perform }.to raise_error(ArgumentError) + end + end + + context 'with fewer than 48 readings for a day' do + let(:config) { create(:amr_data_feed_config, :with_positional_index) } - it 'kind of handles dodgy data' do - expect { converter.perform }.to raise_error(ArgumentError) + let(:readings) do + 46.times.collect { |hh| create_reading_for_period(config, '1710035168313', '25/08/2019', (hh + 1).to_s, ['14.4']) } + end + + it 'rejects the row' do + expect(converter.perform).to be_empty + end end end - context 'more than 48 readings' do - let(:amr_data_feed_config) { create(:amr_data_feed_config, row_per_reading: true, positional_index: true) } + context 'with more than 48 readings per day' do + let(:config) { create(:amr_data_feed_config, :with_positional_index) } let(:readings) do data = [] - 49.times { |idx| data << { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: (idx + 1).to_s, :reading_date => '25/08/2019', :readings => ['14.4'] } } - 48.times { |idx| data << { :amr_data_feed_config_id => 6, :mpan_mprn => '1710035168313', period: (idx + 1).to_s, :reading_date => '26/08/2019', :readings => ['7'] } } + 49.times { |hh| data << create_reading_for_period(config, '1710035168313', '25/08/2019', (hh + 1).to_s, ['14.4']) } + 48.times { |hh| data << create_reading_for_period(config, '1710035168313', '26/08/2019', (hh + 1).to_s, ['14.4']) } data end diff --git a/spec/system/admin/amr_uploaded_readings_spec.rb b/spec/system/admin/amr_uploaded_readings_spec.rb index 5d7cf6c616..72a64ddfe9 100644 --- a/spec/system/admin/amr_uploaded_readings_spec.rb +++ b/spec/system/admin/amr_uploaded_readings_spec.rb @@ -60,7 +60,7 @@ end context 'with row per reading config' do - let!(:config) { create(:amr_data_feed_config, row_per_reading: true, reading_fields: ['Reading']) } + let!(:config) { create(:amr_data_feed_config, :with_reading_time_field) } it 'explains this to user' do expect(page).to have_content('One row per half hour reading') @@ -71,11 +71,11 @@ end it 'explains the reading field column' do - expect(page).to have_content('A reading field column labelled Reading') + expect(page).to have_content('A reading field column labelled ' + config.reading_fields.first) end context 'with a positional index' do - let!(:config) { create(:amr_data_feed_config, row_per_reading: true, reading_fields: ['Reading'], positional_index: true, period_field: 'SettlementTime') } + let!(:config) { create(:amr_data_feed_config, :with_positional_index, period_field: 'SettlementTime') } it 'explains the index column' do expect(page).to have_content('A numbered half-hourly period in a column labelled SettlementTime, e.g. 1, 2, 3, 4') @@ -83,7 +83,7 @@ end context 'with a separate reading time column' do - let!(:config) { create(:amr_data_feed_config, row_per_reading: true, reading_fields: ['Reading'], reading_time_field: 'ReadingTime') } + let!(:config) { create(:amr_data_feed_config, :with_reading_time_field, reading_time_field: 'ReadingTime') } it 'explains the reading time column' do expect(page).to have_content('The reading times to specified in a separate column labelled ReadingTime') @@ -102,7 +102,22 @@ end describe 'normal file format' do - let!(:config) { create(:amr_data_feed_config) } + let!(:config) do + create(:amr_data_feed_config, + description: 'BANES', + mpan_mprn_field: 'M1_Code1', + reading_date_field: 'Date', + date_format: '%b %e %Y %I:%M%p', + number_of_header_rows: 1, + provider_id_field: 'ID', + total_field: 'Total', + meter_description_field: 'Location', + postcode_field: 'PostCode', + units_field: 'Units', + header_example: 'ID,Date,Location,Type,PostCode,Units,Total Units,[00:30],[01:00],[01:30],[02:00],[02:30],[03:00],[03:30],[04:00],[04:30],[05:00],[05:30],[06:00],[06:30],[07:00],[07:30],[08:00],[08:30],[09:00],[09:30],[10:00],[10:30],[11:00],[11:30],[12:00],[12:30],[13:00],[13:30],[14:00],[14:30],[15:00],[15:30],[16:00],[16:30],[17:00],[17:30],[18:00],[18:30],[19:00],[19:30],[20:00],[20:30],[21:00],[21:30],[22:00],[22:30],[23:00],[23:30],[24:00],M1_Code1,M1_Code2', + reading_fields: '[00:30],[01:00],[01:30],[02:00],[02:30],[03:00],[03:30],[04:00],[04:30],[05:00],[05:30],[06:00],[06:30],[07:00],[07:30],[08:00],[08:30],[09:00],[09:30],[10:00],[10:30],[11:00],[11:30],[12:00],[12:30],[13:00],[13:30],[14:00],[14:30],[15:00],[15:30],[16:00],[16:30],[17:00],[17:30],[18:00],[18:30],[19:00],[19:30],[20:00],[20:30],[21:00],[21:30],[22:00],[22:30],[23:00],[23:30],[24:00]'.split(',') + ) + end before do sign_in(admin)