From e20bd4a784eba194d5c32d1d9cb4d27b85aadebf Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 24 Oct 2023 19:50:37 -0400 Subject: [PATCH 1/9] fix: switch to column E for schedule, H for RT Also, minor fixes, plus ternary operator if one URL is empty, use the other. --- scripts/process_csv_in_github_action.swift | 89 +++++++++++----------- 1 file changed, 46 insertions(+), 43 deletions(-) diff --git a/scripts/process_csv_in_github_action.swift b/scripts/process_csv_in_github_action.swift index abb76f01..6f087e43 100644 --- a/scripts/process_csv_in_github_action.swift +++ b/scripts/process_csv_in_github_action.swift @@ -4,35 +4,35 @@ import Foundation #endif enum column : Int, CaseIterable { - case timestamp = 0 - case provider = 1 - case regioncity = 2 - case currenturl = 3 - case updatednewsourceurl = 4 - case datatype1 = 5 - case request = 6 - case downloadurl = 7 - case country = 8 - case subdivision_name = 9 - case municipality = 10 - case name = 11 - case yournameorg = 12 - case license_url = 13 - case tripupdatesurl = 14 - case servicealertsurl = 15 - case genunknownrturl = 16 - case authentication_type = 17 - case authentication_info_url = 18 - case api_key_parameter_name = 19 - case note = 20 - case gtfsschedulefeatures = 21 - case gtfsschedulestatus = 22 - case gtfsrealtimestatus = 23 - case youremail = 24 - case dataproduceremail = 25 - case realtimefeatures = 26 - case isocountrycode = 27 - case feedupdatestatus = 28 + case timestamp = 0 // A + case provider = 1 // B + case regioncity = 2 // C + case currenturl = 3 // D + case updatednewsourceurl = 4 // E + case datatype = 5 // F + case request = 6 // G + case downloadurl = 7 // H + case country = 8 // I + case subdivision_name = 9 // J + case municipality = 10 // K + case name = 11 // L + case yournameorg = 12 // M + case license_url = 13 // N + case tripupdatesurl = 14 // O + case servicealertsurl = 15 // P + case genunknownrturl = 16 // Q + case authentication_type = 17 // R + case authentication_info_url = 18 // S + case api_key_parameter_name = 19 // T + case note = 20 // U + case gtfsschedulefeatures = 21 // W + case gtfsschedulestatus = 22 // Y + case gtfsrealtimestatus = 23 // Z + case youremail = 24 // AA + case dataproduceremail = 25 // AB + case realtimefeatures = 26 // AC + case isocountrycode = 27 // AB + case feedupdatestatus = 28 // AC } enum defaults: String { @@ -89,7 +89,7 @@ if CommandLine.argc == 5 { let timestamp : String = line[column.timestamp.rawValue].trimmingCharacters(in: .whitespacesAndNewlines) let provider : String = line[column.provider.rawValue] - let datatype1 : String = line[column.datatype1.rawValue] + let datatype : String = line[column.datatype.rawValue] let request : String = line[column.request.rawValue] let country : String = line[column.country.rawValue] let subdivision_name : String = line[column.subdivision_name.rawValue] @@ -97,6 +97,7 @@ if CommandLine.argc == 5 { let name : String = line[column.name.rawValue] let license_url : String = line[column.license_url.rawValue] let downloadURL : String = line[column.downloadurl.rawValue] + let updatednewsourceurl : String = line[column.updatednewsourceurl.rawValue] let authentication_type : String = line[column.authentication_type.rawValue] let authentication_info_url : String = line[column.authentication_info_url.rawValue] let api_key_parameter_name : String = line[column.api_key_parameter_name.rawValue] @@ -106,49 +107,51 @@ if CommandLine.argc == 5 { let gtfsrealtimestatus : String = line[column.gtfsrealtimestatus.rawValue] let realtimefeatures : String = line[column.realtimefeatures.rawValue] + + let dateFromCurrentLine : String = extractDate(from: timestamp, usingGREP: dateFormatAsRegex, desiredDateFormat: dateFormatDesiredArg) if dateFromCurrentLine == dateToFind { // ...the row has been added on the date we're looking for, process it. if request.contains(requestType.isAddNewFeed.rawValue) { // add new feed - if datatype1.contains(dataType.schedule.rawValue) { // add_gtfs_schedule_source + if datatype.contains(dataType.schedule.rawValue) { // add_gtfs_schedule_source - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(provider), country_code=\(country), direct_download_url=\(downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\(license_url), name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(provider), country_code=\(country), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\(license_url), name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" - } else if datatype1.contains(dataType.realtime.rawValue) { // add_gtfs_realtime_source + } else if datatype.contains(dataType.realtime.rawValue) { // add_gtfs_realtime_source // Emma: entity_type matches the realtime Data type options of Vehicle Positions, Trip Updates, or Service Alerts. If one of those three are selected, add it. If not, omit it. - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(datatype1), provider=\(provider), direct_download_url=\(downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(datatype), provider=\(provider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } } else if request.contains(requestType.isUpdateExistingFeed.rawValue) { // update existing feed - if datatype1.contains(dataType.schedule.rawValue) { // update_gtfs_schedule_source + if datatype.contains(dataType.schedule.rawValue) { // update_gtfs_schedule_source - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(provider), name=\(name), country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(provider), name=\(name), country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" - } else if datatype1.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source + } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\(datatype1), provider=\(provider), direct_download_url=\(downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\(datatype), provider=\(provider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } } else if request.contains(requestType.isToRemoveFeed.rawValue) { // remove feed - if datatype1.contains(dataType.schedule.rawValue) { // update_gtfs_schedule_source + if datatype.contains(dataType.schedule.rawValue) { // update_gtfs_schedule_source - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(provider), name=\"**** Requested for removal ****\", country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(provider), name=\"**** Requested for removal ****\", country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" - } else if datatype1.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source + } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\(datatype1), provider=\(provider), direct_download_url=\(downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\"**** Requested for removal ****\", static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\(datatype), provider=\(provider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\"**** Requested for removal ****\", static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } } else { // ... assume this is a new feed by default :: add_gtfs_schedule_source - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(provider), country_code=\(country), direct_download_url=\(downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\(license_url), name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(provider), country_code=\(country), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\(license_url), name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" } From 0437c3b3072c33cce8fee56c60b503862ce05fda Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 24 Oct 2023 20:03:43 -0400 Subject: [PATCH 2/9] Fix for data type for real time Needed to provide an array to the entity_type value (vp, tu, sa, default ug). --- scripts/process_csv_in_github_action.swift | 34 ++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/scripts/process_csv_in_github_action.swift b/scripts/process_csv_in_github_action.swift index 6f087e43..13963fbe 100644 --- a/scripts/process_csv_in_github_action.swift +++ b/scripts/process_csv_in_github_action.swift @@ -50,6 +50,20 @@ enum dataType: String { case realtime = "Realtime" } +enum realtimeDataType: String { + case vehiclePositions = "Vehicle Positions" + case tripUpdates = "Trip Updates" + case serviceAlerts = "Service Alerts" + case unknown = "general / unknown" +} + +enum realtimeDataTypeCode: String { + case vehiclePositions = "vp" + case tripUpdates = "tu" + case serviceAlerts = "sa" + case unknown = "gu" +} + let arguments : [String] = CommandLine.arguments if CommandLine.argc == 5 { @@ -122,7 +136,8 @@ if CommandLine.argc == 5 { } else if datatype.contains(dataType.realtime.rawValue) { // add_gtfs_realtime_source // Emma: entity_type matches the realtime Data type options of Vehicle Positions, Trip Updates, or Service Alerts. If one of those three are selected, add it. If not, omit it. - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(datatype), provider=\(provider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + let realtimecode : String = realtimeCode(for:datatype) + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(provider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } @@ -133,8 +148,9 @@ if CommandLine.argc == 5 { PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(provider), name=\(name), country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source - - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\(datatype), provider=\(provider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + + let realtimecode : String = realtimeCode(for:datatype) + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\(realtimecode), provider=\(provider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } } else if request.contains(requestType.isToRemoveFeed.rawValue) { // remove feed @@ -144,8 +160,9 @@ if CommandLine.argc == 5 { PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(provider), name=\"**** Requested for removal ****\", country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source - - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\(datatype), provider=\(provider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\"**** Requested for removal ****\", static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + + let realtimecode : String = realtimeCode(for:datatype) + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\"[\(realtimecode)]\", provider=\(provider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\"**** Requested for removal ****\", static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } @@ -191,4 +208,11 @@ func extractDate(from theDateToConvert: String, usingGREP dateFormatAsGREP: Rege // return default date return defaults.date.rawValue +} + +func realtimeCode(for theDataType: String) -> String { + if theDataType.contains(realtimeDataType.vehiclePositions.rawValue) { return realtimeDataTypeCode.vehiclePositions.rawValue } + if theDataType.contains(realtimeDataType.tripUpdates.rawValue) { return realtimeDataTypeCode.tripUpdates.rawValue } + if theDataType.contains(realtimeDataType.serviceAlerts.rawValue) { return realtimeDataTypeCode.serviceAlerts.rawValue } + return realtimeDataTypeCode.unknown.rawValue } \ No newline at end of file From 8a6f7f4abcf10155a40722b2d3d4c2d39a7dfeb8 Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 24 Oct 2023 20:08:09 -0400 Subject: [PATCH 3/9] fix: missing realtime type when column G is empty --- scripts/process_csv_in_github_action.swift | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/process_csv_in_github_action.swift b/scripts/process_csv_in_github_action.swift index 13963fbe..05e24aa2 100644 --- a/scripts/process_csv_in_github_action.swift +++ b/scripts/process_csv_in_github_action.swift @@ -167,9 +167,17 @@ if CommandLine.argc == 5 { } } else { // ... assume this is a new feed by default :: add_gtfs_schedule_source - - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(provider), country_code=\(country), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\(license_url), name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" - + + if datatype.contains(dataType.schedule.rawValue) { // update_gtfs_schedule_source + + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(provider), country_code=\(country), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\(license_url), name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + + } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source + + let realtimecode : String = realtimeCode(for:datatype) + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(provider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + + } } } From e22d231f3f7d16bba9f633494c33322415d48ac6 Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 24 Oct 2023 20:20:02 -0400 Subject: [PATCH 4/9] fix: added logic when provider cell empty Store last known provider (with default to `TO_BE_PROVIDED`), and use it as a suggestion if the cell is empty. --- scripts/process_csv_in_github_action.swift | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/scripts/process_csv_in_github_action.swift b/scripts/process_csv_in_github_action.swift index 05e24aa2..d20eeba6 100644 --- a/scripts/process_csv_in_github_action.swift +++ b/scripts/process_csv_in_github_action.swift @@ -37,6 +37,7 @@ enum column : Int, CaseIterable { enum defaults: String { case date = "01/01/1970" + case toBeProvided = "TO_BE_PROVIDED" } enum requestType: String { @@ -93,6 +94,7 @@ if CommandLine.argc == 5 { } var PYTHON_SCRIPT_OUTPUT : String = "" + var lastKnownProvider : String = defaults.toBeProvided.rawValue let dateFormatAsRegex : Regex = try Regex(dateFormatGREPArg) for line : [String] in csvArray { @@ -121,7 +123,8 @@ if CommandLine.argc == 5 { let gtfsrealtimestatus : String = line[column.gtfsrealtimestatus.rawValue] let realtimefeatures : String = line[column.realtimefeatures.rawValue] - + if provider.count > 0 { lastKnownProvider = provider } // logic: if the cell is empty, suggest last known provider. + let finalProvider : String = provider.isEmpty ? "\(defaults.toBeProvided.rawValue) (\(lastKnownProvider) ?)" : provider let dateFromCurrentLine : String = extractDate(from: timestamp, usingGREP: dateFormatAsRegex, desiredDateFormat: dateFormatDesiredArg) @@ -130,14 +133,13 @@ if CommandLine.argc == 5 { if request.contains(requestType.isAddNewFeed.rawValue) { // add new feed if datatype.contains(dataType.schedule.rawValue) { // add_gtfs_schedule_source - - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(provider), country_code=\(country), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\(license_url), name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(finalProvider), country_code=\(country), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\(license_url), name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" } else if datatype.contains(dataType.realtime.rawValue) { // add_gtfs_realtime_source // Emma: entity_type matches the realtime Data type options of Vehicle Positions, Trip Updates, or Service Alerts. If one of those three are selected, add it. If not, omit it. let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(provider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } @@ -145,24 +147,24 @@ if CommandLine.argc == 5 { if datatype.contains(dataType.schedule.rawValue) { // update_gtfs_schedule_source - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(provider), name=\(name), country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(finalProvider), name=\(name), country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\(realtimecode), provider=\(provider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } } else if request.contains(requestType.isToRemoveFeed.rawValue) { // remove feed if datatype.contains(dataType.schedule.rawValue) { // update_gtfs_schedule_source - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(provider), name=\"**** Requested for removal ****\", country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(finalProvider), name=\"**** Requested for removal ****\", country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\"[\(realtimecode)]\", provider=\(provider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\"**** Requested for removal ****\", static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\"[\(realtimecode)]\", provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\"**** Requested for removal ****\", static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } @@ -170,12 +172,12 @@ if CommandLine.argc == 5 { if datatype.contains(dataType.schedule.rawValue) { // update_gtfs_schedule_source - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(provider), country_code=\(country), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\(license_url), name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(finalProvider), country_code=\(country), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\(license_url), name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(provider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } } From 3be05f303d23c8e0c82724ce16a46d37404eaa67 Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 24 Oct 2023 20:22:51 -0400 Subject: [PATCH 5/9] fix: Used constant instead of plain string --- scripts/process_csv_in_github_action.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/process_csv_in_github_action.swift b/scripts/process_csv_in_github_action.swift index d20eeba6..2d48817b 100644 --- a/scripts/process_csv_in_github_action.swift +++ b/scripts/process_csv_in_github_action.swift @@ -139,7 +139,7 @@ if CommandLine.argc == 5 { // Emma: entity_type matches the realtime Data type options of Vehicle Positions, Trip Updates, or Service Alerts. If one of those three are selected, add it. If not, omit it. let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"\(defaults.toBeProvided.rawValue)\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } @@ -152,7 +152,7 @@ if CommandLine.argc == 5 { } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"\(defaults.toBeProvided.rawValue)", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } } else if request.contains(requestType.isToRemoveFeed.rawValue) { // remove feed @@ -164,7 +164,7 @@ if CommandLine.argc == 5 { } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\"[\(realtimecode)]\", provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\"**** Requested for removal ****\", static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\"[\(realtimecode)]\", provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\"**** Requested for removal ****\", static_reference=\"\(defaults.toBeProvided.rawValue)\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } @@ -177,7 +177,7 @@ if CommandLine.argc == 5 { } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"TO_BE_PROVIDED\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"\(defaults.toBeProvided.rawValue)\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } } From a2e46b0e8389a265b3cf26846de2b577308cb014 Mon Sep 17 00:00:00 2001 From: Fred Date: Fri, 27 Oct 2023 21:57:41 -0400 Subject: [PATCH 6/9] fix: `license_url` between quotes Possible fix for this error: ``` File "", line 1 from tools.operations import *; update_gtfs_realtime_source(mdb_source_id="", entity_type="GTFS Realtime - Trip Updates", provider="Arlington Transit", direct_download_url="https://realtime.arlingtontransit.com/gtfsrt/trips", authentication_type="", authentication_info_url="0 or (empty) - No authentication required.", api_key_parameter_name="", license_url= Redmon Group Inc", name="", static_reference="TO_BE_PROVIDED", note="", status="", features="Active") ^ SyntaxError: unterminated string literal (detected at line 1) Error: Process completed with exit code 1. ``` --- scripts/process_csv_in_github_action.swift | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/process_csv_in_github_action.swift b/scripts/process_csv_in_github_action.swift index 2d48817b..00fc47cb 100644 --- a/scripts/process_csv_in_github_action.swift +++ b/scripts/process_csv_in_github_action.swift @@ -133,13 +133,13 @@ if CommandLine.argc == 5 { if request.contains(requestType.isAddNewFeed.rawValue) { // add new feed if datatype.contains(dataType.schedule.rawValue) { // add_gtfs_schedule_source - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(finalProvider), country_code=\(country), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\(license_url), name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(finalProvider), country_code=\(country), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\"\(license_url)\", name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" } else if datatype.contains(dataType.realtime.rawValue) { // add_gtfs_realtime_source // Emma: entity_type matches the realtime Data type options of Vehicle Positions, Trip Updates, or Service Alerts. If one of those three are selected, add it. If not, omit it. let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"\(defaults.toBeProvided.rawValue)\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\"\(license_url)\", name=\(name), static_reference=\"\(defaults.toBeProvided.rawValue)\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } @@ -147,24 +147,24 @@ if CommandLine.argc == 5 { if datatype.contains(dataType.schedule.rawValue) { // update_gtfs_schedule_source - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(finalProvider), name=\(name), country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(finalProvider), name=\(name), country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\"\(license_url)\", status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"\(defaults.toBeProvided.rawValue)", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\"\(license_url)\", name=\(name), static_reference=\"\(defaults.toBeProvided.rawValue)", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } } else if request.contains(requestType.isToRemoveFeed.rawValue) { // remove feed if datatype.contains(dataType.schedule.rawValue) { // update_gtfs_schedule_source - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(finalProvider), name=\"**** Requested for removal ****\", country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(finalProvider), name=\"**** Requested for removal ****\", country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\"\(license_url)\", status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\"[\(realtimecode)]\", provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\"**** Requested for removal ****\", static_reference=\"\(defaults.toBeProvided.rawValue)\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\"[\(realtimecode)]\", provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\"\(license_url)\", name=\"**** Requested for removal ****\", static_reference=\"\(defaults.toBeProvided.rawValue)\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } @@ -172,12 +172,12 @@ if CommandLine.argc == 5 { if datatype.contains(dataType.schedule.rawValue) { // update_gtfs_schedule_source - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(finalProvider), country_code=\(country), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\(license_url), name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(finalProvider), country_code=\(country), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\"\(license_url)\", name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\(license_url), name=\(name), static_reference=\"\(defaults.toBeProvided.rawValue)\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\"\(license_url)\", name=\(name), static_reference=\"\(defaults.toBeProvided.rawValue)\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" } } From 1feb2fe06b089ba0d90fde918be2c1207b45c024 Mon Sep 17 00:00:00 2001 From: Fred Date: Sat, 28 Oct 2023 10:59:41 -0400 Subject: [PATCH 7/9] fix: if license_url is invalid, provide context --- scripts/process_csv_in_github_action.swift | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/scripts/process_csv_in_github_action.swift b/scripts/process_csv_in_github_action.swift index 00fc47cb..ccd8a13e 100644 --- a/scripts/process_csv_in_github_action.swift +++ b/scripts/process_csv_in_github_action.swift @@ -111,7 +111,7 @@ if CommandLine.argc == 5 { let subdivision_name : String = line[column.subdivision_name.rawValue] let municipality : String = line[column.municipality.rawValue] let name : String = line[column.name.rawValue] - let license_url : String = line[column.license_url.rawValue] + var license_url : String = line[column.license_url.rawValue] let downloadURL : String = line[column.downloadurl.rawValue] let updatednewsourceurl : String = line[column.updatednewsourceurl.rawValue] let authentication_type : String = line[column.authentication_type.rawValue] @@ -123,9 +123,14 @@ if CommandLine.argc == 5 { let gtfsrealtimestatus : String = line[column.gtfsrealtimestatus.rawValue] let realtimefeatures : String = line[column.realtimefeatures.rawValue] - if provider.count > 0 { lastKnownProvider = provider } // logic: if the cell is empty, suggest last known provider. + // Check if provider is empty, suggest last known if true. + if provider.count > 0 { lastKnownProvider = provider } let finalProvider : String = provider.isEmpty ? "\(defaults.toBeProvided.rawValue) (\(lastKnownProvider) ?)" : provider + // Check if license URL is valid + let urlPresent : Bool = isURLPresent(in: license_url) + if ( urlPresent == false && license_url.count > 0 ) { license_url = "INVALID_OR_NO_URL_PROVIDED : [ \(license_url) ]" } + let dateFromCurrentLine : String = extractDate(from: timestamp, usingGREP: dateFormatAsRegex, desiredDateFormat: dateFormatDesiredArg) if dateFromCurrentLine == dateToFind { // ...the row has been added on the date we're looking for, process it. @@ -225,4 +230,16 @@ func realtimeCode(for theDataType: String) -> String { if theDataType.contains(realtimeDataType.tripUpdates.rawValue) { return realtimeDataTypeCode.tripUpdates.rawValue } if theDataType.contains(realtimeDataType.serviceAlerts.rawValue) { return realtimeDataTypeCode.serviceAlerts.rawValue } return realtimeDataTypeCode.unknown.rawValue +} + +func isURLPresent(in string: String) -> Bool { + do { + let pattern : String = "(http|https)://[a-zA-Z0-9./?=_%:-]*" + let regex = try NSRegularExpression(pattern: pattern, options: []) + let range = NSRange(location: 0, length: string.utf16.count) + if let _ = regex.firstMatch(in: string, options: [], range: range) { return true } + } catch { + print("Error creating or using regular expression: \(error)") + } + return false } \ No newline at end of file From c65b2903c5341282861216cfc50f17df3d3840ff Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 7 Nov 2023 09:13:50 -0500 Subject: [PATCH 8/9] Fix: add pip upgrade to remove warning in log --- .github/workflows/add_new_or_updated_feeds.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/add_new_or_updated_feeds.yml b/.github/workflows/add_new_or_updated_feeds.yml index 929719d2..28912a45 100644 --- a/.github/workflows/add_new_or_updated_feeds.yml +++ b/.github/workflows/add_new_or_updated_feeds.yml @@ -87,6 +87,7 @@ jobs: run: | python -m venv env source env/bin/activate + pip install --upgrade pip pip install virtualenv --quiet pip install gtfs_kit --quiet pip install unidecode --quiet From 34919e17e53ee87afff6472c42c92192092eeed7 Mon Sep 17 00:00:00 2001 From: Fred Date: Tue, 7 Nov 2023 15:17:47 -0500 Subject: [PATCH 9/9] fix: changes to correct errors in Python script --- scripts/process_csv_in_github_action.swift | 46 +++++++++++++--------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/scripts/process_csv_in_github_action.swift b/scripts/process_csv_in_github_action.swift index ccd8a13e..c903cfc0 100644 --- a/scripts/process_csv_in_github_action.swift +++ b/scripts/process_csv_in_github_action.swift @@ -129,7 +129,7 @@ if CommandLine.argc == 5 { // Check if license URL is valid let urlPresent : Bool = isURLPresent(in: license_url) - if ( urlPresent == false && license_url.count > 0 ) { license_url = "INVALID_OR_NO_URL_PROVIDED : [ \(license_url) ]" } + if ( urlPresent == false && license_url.count > 0 ) { license_url = "INVALID_OR_NO_URL_PROVIDED" } let dateFromCurrentLine : String = extractDate(from: timestamp, usingGREP: dateFormatAsRegex, desiredDateFormat: dateFormatDesiredArg) @@ -138,13 +138,15 @@ if CommandLine.argc == 5 { if request.contains(requestType.isAddNewFeed.rawValue) { // add new feed if datatype.contains(dataType.schedule.rawValue) { // add_gtfs_schedule_source - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(finalProvider), country_code=\(country), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\"\(license_url)\", name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + let authType : Int = authenticationType(for: authentication_type) + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\"\(finalProvider)\", country_code=\"\(country)\", direct_download_url=\"\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl)\", authentication_type=\(authType), authentication_info_url=\"\(authentication_info_url)\", api_key_parameter_name=\"\(api_key_parameter_name)\", subdivision_name=\"\(subdivision_name)\", municipality=\"\(municipality)\", license_url=\"\(license_url)\", name=\"\(name)\", status=\"\(gtfsschedulestatus)\", features=\"\(gtfsschedulefeatures)\")" } else if datatype.contains(dataType.realtime.rawValue) { // add_gtfs_realtime_source // Emma: entity_type matches the realtime Data type options of Vehicle Positions, Trip Updates, or Service Alerts. If one of those three are selected, add it. If not, omit it. + let authType : Int = authenticationType(for: authentication_type) let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\"\(license_url)\", name=\(name), static_reference=\"\(defaults.toBeProvided.rawValue)\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\"\(realtimecode)\", provider=\"\(finalProvider)\", direct_download_url=\"\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL)\", authentication_type=\(authType), authentication_info_url=\"\(authentication_info_url)\", api_key_parameter_name=\"\(api_key_parameter_name)\", license_url=\"\(license_url)\", name=\"\(name)\", note=\"\(note)\", status=\"\(gtfsrealtimestatus)\", features=\"\(realtimefeatures)\")" } @@ -152,24 +154,28 @@ if CommandLine.argc == 5 { if datatype.contains(dataType.schedule.rawValue) { // update_gtfs_schedule_source - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(finalProvider), name=\(name), country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\"\(license_url)\", status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + let authType : Int = authenticationType(for: authentication_type) + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\"\(finalProvider)\", name=\"\(name)\", country_code=\"\(country)\", subdivision_name=\"\(subdivision_name)\", municipality=\"\(municipality)\", direct_download_url=\"\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl)\", authentication_type=\(authType), authentication_info_url=\"\(authentication_info_url)\", api_key_parameter_name=\"\(api_key_parameter_name)\", license_url=\"\(license_url)\", status=\"\(gtfsschedulestatus)\", features=\"\(gtfsschedulefeatures)\")" } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source + let authType : Int = authenticationType(for: authentication_type) let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\"\(license_url)\", name=\(name), static_reference=\"\(defaults.toBeProvided.rawValue)", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\"\(realtimecode)\", provider=\"\(finalProvider)\", direct_download_url=\"\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL)\", authentication_type=\(authType), authentication_info_url=\"\(authentication_info_url)\", api_key_parameter_name=\"\(api_key_parameter_name)\", license_url=\"\(license_url)\", name=\"\(name)\", note=\"\(note)\", status=\"\(gtfsrealtimestatus)\", features=\"\(realtimefeatures)\")" } } else if request.contains(requestType.isToRemoveFeed.rawValue) { // remove feed if datatype.contains(dataType.schedule.rawValue) { // update_gtfs_schedule_source - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\(finalProvider), name=\"**** Requested for removal ****\", country_code=\(country), subdivision_name=\(subdivision_name), municipality=\(municipality), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\"\(license_url)\", status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + let authType : Int = authenticationType(for: authentication_type) + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_schedule_source(mdb_source_id=\"\", provider=\"\(finalProvider)\", name=\"**** Requested for removal ****\", country_code=\"\(country)\", subdivision_name=\"\(subdivision_name)\", municipality=\"\(municipality)\", direct_download_url=\"\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl)\", authentication_type=\(authType), authentication_info_url=\"\(authentication_info_url)\", api_key_parameter_name=\"\(api_key_parameter_name)\", license_url=\"\(license_url)\", status=\"\(gtfsschedulestatus)\", features=\"\(gtfsschedulefeatures)\")" } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source + let authType : Int = authenticationType(for: authentication_type) let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\"[\(realtimecode)]\", provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\"\(license_url)\", name=\"**** Requested for removal ****\", static_reference=\"\(defaults.toBeProvided.rawValue)\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "update_gtfs_realtime_source(mdb_source_id=\"\", entity_type=\"[\(realtimecode)]\", provider=\"\(finalProvider)\", direct_download_url=\"\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL)\", authentication_type=\(authType), authentication_info_url=\"\(authentication_info_url)\", api_key_parameter_name=\"\(api_key_parameter_name)\", license_url=\"\(license_url)\", name=\"**** Requested for removal ****\", note=\"\(note)\", status=\"\(gtfsrealtimestatus)\", features=\"\(realtimefeatures)\")" } @@ -177,12 +183,14 @@ if CommandLine.argc == 5 { if datatype.contains(dataType.schedule.rawValue) { // update_gtfs_schedule_source - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\(finalProvider), country_code=\(country), direct_download_url=\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), subdivision_name=\(subdivision_name), municipality=\(municipality), license_url=\"\(license_url)\", name=\(name), status=\(gtfsschedulestatus), features=\(gtfsschedulefeatures))" + let authType : Int = authenticationType(for: authentication_type) + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_schedule_source(provider=\"\(finalProvider)\", country_code=\"\(country)\", direct_download_url=\"\(updatednewsourceurl.isEmpty ? downloadURL : updatednewsourceurl)\", authentication_type=\(authType), authentication_info_url=\"\(authentication_info_url)\", api_key_parameter_name=\"\(api_key_parameter_name)\", subdivision_name=\"\(subdivision_name)\", municipality=\"\(municipality)\", license_url=\"\(license_url)\", name=\"\(name)\", status=\"\(gtfsschedulestatus)\", features=\"\(gtfsschedulefeatures)\")" } else if datatype.contains(dataType.realtime.rawValue) { // update_gtfs_realtime_source + let authType : Int = authenticationType(for: authentication_type) let realtimecode : String = realtimeCode(for:datatype) - PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\(realtimecode), provider=\(finalProvider), direct_download_url=\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL), authentication_type=\(authentication_type), authentication_info_url=\(authentication_info_url), api_key_parameter_name=\(api_key_parameter_name), license_url=\"\(license_url)\", name=\(name), static_reference=\"\(defaults.toBeProvided.rawValue)\", note=\(note), status=\(gtfsrealtimestatus), features=\(realtimefeatures))" + PYTHON_SCRIPT_ARGS_TEMP = "add_gtfs_realtime_source(entity_type=\"\(realtimecode)\", provider=\"\(finalProvider)\", direct_download_url=\"\(downloadURL.isEmpty ? updatednewsourceurl : downloadURL)\", authentication_type=\(authType), authentication_info_url=\"\(authentication_info_url)\", api_key_parameter_name=\"\(api_key_parameter_name)\", license_url=\"\(license_url)\", name=\"\(name)\", note=\"\(note)\", status=\"\(gtfsrealtimestatus)\", features=\"\(realtimefeatures)\")" } } @@ -225,21 +233,23 @@ func extractDate(from theDateToConvert: String, usingGREP dateFormatAsGREP: Rege return defaults.date.rawValue } +func authenticationType(for authString: String) -> Int { + if authString.contains("0") { return 0 } + if authString.contains("1") { return 1 } + if authString.contains("2") { return 2 } + return 0 +} + func realtimeCode(for theDataType: String) -> String { if theDataType.contains(realtimeDataType.vehiclePositions.rawValue) { return realtimeDataTypeCode.vehiclePositions.rawValue } if theDataType.contains(realtimeDataType.tripUpdates.rawValue) { return realtimeDataTypeCode.tripUpdates.rawValue } if theDataType.contains(realtimeDataType.serviceAlerts.rawValue) { return realtimeDataTypeCode.serviceAlerts.rawValue } - return realtimeDataTypeCode.unknown.rawValue + return realtimeDataTypeCode.tripUpdates.rawValue } func isURLPresent(in string: String) -> Bool { - do { - let pattern : String = "(http|https)://[a-zA-Z0-9./?=_%:-]*" - let regex = try NSRegularExpression(pattern: pattern, options: []) - let range = NSRange(location: 0, length: string.utf16.count) - if let _ = regex.firstMatch(in: string, options: [], range: range) { return true } - } catch { - print("Error creating or using regular expression: \(error)") - } + let pattern : String = #"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)"# + let range = string.range(of: pattern, options: .regularExpression) + if range != nil { return true } return false } \ No newline at end of file