From e88ddb98e0a80393252d87621046e2dedb350ad6 Mon Sep 17 00:00:00 2001 From: Arkadi Chubrik Date: Sun, 19 May 2019 01:06:13 +0300 Subject: [PATCH] Get all relations mode --- ...rdsExtensions.cs => GeoPointExtensions.cs} | 0 OsmDataKit/Models/OsmResponse.cs | 2 + OsmDataKit/OsmService.cs | 112 ++++++++++++++---- 3 files changed, 94 insertions(+), 20 deletions(-) rename OsmDataKit/Extensions/{GeoCoordsExtensions.cs => GeoPointExtensions.cs} (100%) diff --git a/OsmDataKit/Extensions/GeoCoordsExtensions.cs b/OsmDataKit/Extensions/GeoPointExtensions.cs similarity index 100% rename from OsmDataKit/Extensions/GeoCoordsExtensions.cs rename to OsmDataKit/Extensions/GeoPointExtensions.cs diff --git a/OsmDataKit/Models/OsmResponse.cs b/OsmDataKit/Models/OsmResponse.cs index f5aae81..1950729 100644 --- a/OsmDataKit/Models/OsmResponse.cs +++ b/OsmDataKit/Models/OsmResponse.cs @@ -40,5 +40,7 @@ public RelationObject[] _relations get => Relations.Values.ToArray(); set => Relations = value.ToDictionary(i => i.Id); } + + internal Dictionary AllRelations { get; set; } } } diff --git a/OsmDataKit/OsmService.cs b/OsmDataKit/OsmService.cs index 33f8df6..c46b018 100644 --- a/OsmDataKit/OsmService.cs +++ b/OsmDataKit/OsmService.cs @@ -70,7 +70,10 @@ public static void ValidateSource(string pbfPath) #region Load - public static OsmResponse Load(string pbfPath, Func filter) + public static OsmResponse Load(string pbfPath, Func filter) => + Load(pbfPath, filter, getAllRelations: false); + + private static OsmResponse Load(string pbfPath, Func filter, bool getAllRelations) { Debug.Assert(pbfPath != null); Debug.Assert(filter != null); @@ -85,24 +88,38 @@ public static OsmResponse Load(string pbfPath, Func filter) var foundNodes = new Dictionary(); var foundWays = new Dictionary(); var foundRelations = new Dictionary(); + var allRelations = getAllRelations ? new Dictionary() : null; using (var fileStream = FileClient.OpenRead(pbfPath)) { var source = new PBFOsmStreamSource(fileStream); - foreach (var osmGeo in source.Where(filter)) + foreach (var osmGeo in source) switch (osmGeo.Type) { case OsmGeoType.Node: - foundNodes.Add(osmGeo.Id.GetValueOrDefault(), new NodeObject(osmGeo as Node)); + + if (filter(osmGeo)) + foundNodes.Add(osmGeo.Id.GetValueOrDefault(), new NodeObject(osmGeo as Node)); + break; case OsmGeoType.Way: - foundWays.Add(osmGeo.Id.GetValueOrDefault(), new WayObject(osmGeo as Way)); + + if (filter(osmGeo)) + foundWays.Add(osmGeo.Id.GetValueOrDefault(), new WayObject(osmGeo as Way)); + break; case OsmGeoType.Relation: - foundRelations.Add(osmGeo.Id.GetValueOrDefault(), new RelationObject(osmGeo as Relation)); + RelationObject relation = null; + + if (getAllRelations) + allRelations.Add(osmGeo.Id.GetValueOrDefault(), relation = new RelationObject(osmGeo as Relation)); + + if (filter(osmGeo)) + foundRelations.Add(osmGeo.Id.GetValueOrDefault(), relation ?? new RelationObject(osmGeo as Relation)); + break; default: @@ -112,10 +129,13 @@ public static OsmResponse Load(string pbfPath, Func filter) LogService.LogInfo($"Loaded: {foundNodes.Count} nodes, {foundWays.Count} ways, {foundRelations.Count} relations"); LogService.EndInfo("Load OSM data completed"); - return new OsmResponse { Nodes = foundNodes, Ways = foundWays, Relations = foundRelations }; + return new OsmResponse { Nodes = foundNodes, Ways = foundWays, Relations = foundRelations, AllRelations = allRelations }; } - public static OsmResponse Load(string pbfPath, OsmRequest request) + public static OsmResponse Load(string pbfPath, OsmRequest request) => + Load(pbfPath, request, getAllRelations: false); + + private static OsmResponse Load(string pbfPath, OsmRequest request, bool getAllRelations) { Debug.Assert(pbfPath != null); Debug.Assert(request != null); @@ -143,6 +163,7 @@ public static OsmResponse Load(string pbfPath, OsmRequest request) var foundNodes = new Dictionary(); var foundWays = new Dictionary(); var foundRelations = new Dictionary(); + var allRelations = getAllRelations ? new Dictionary() : null; List missedNodeIds = null; List missedWayIds = null; List missedRelationIds = null; @@ -190,7 +211,7 @@ public static OsmResponse Load(string pbfPath, OsmRequest request) LogService.LogInfo("Loaded 0 ways"); - if (requestRelationIds.Count > 0) + if (requestRelationIds.Count > 0 || getAllRelations) { thisType = OsmGeoType.Relation; continue; @@ -226,7 +247,7 @@ public static OsmResponse Load(string pbfPath, OsmRequest request) else LogService.LogWarning($"{logMessage} ({missedWayIds.Count} missed)"); - if (requestRelationIds.Count > 0) + if (requestRelationIds.Count > 0 || getAllRelations) { thisType = OsmGeoType.Relation; continue; @@ -240,12 +261,16 @@ public static OsmResponse Load(string pbfPath, OsmRequest request) continue; id = osmGeo.Id.GetValueOrDefault(); + RelationObject relation = null; + + if (getAllRelations) + allRelations.Add(id, relation = new RelationObject(osmGeo as Relation)); if (requestRelationIds.Contains(id)) { - foundRelations.Add(id, new RelationObject(osmGeo as Relation)); + foundRelations.Add(id, relation ?? new RelationObject(osmGeo as Relation)); - if (foundRelations.Count == requestRelationIds.Count) + if (!getAllRelations && foundRelations.Count == requestRelationIds.Count) goto Complete; } @@ -275,7 +300,8 @@ public static OsmResponse Load(string pbfPath, OsmRequest request) Relations = foundRelations, MissedNodeIds = missedNodeIds ?? new List(0), MissedWayIds = missedWayIds ?? new List(0), - MissedRelationIds = missedRelationIds ?? new List(0) + MissedRelationIds = missedRelationIds ?? new List(0), + AllRelations = allRelations }; } @@ -283,28 +309,30 @@ public static OsmResponse Load(string pbfPath, OsmRequest request) #region Load objects - public static OsmObjectResponse LoadObjects(string pbfPath, string cacheName, OsmRequest request, int stepLimit = 0) + public static OsmObjectResponse LoadObjects( + string pbfPath, string cacheName, OsmRequest request, int stepLimit = 0, bool lowMemoryMode = false) { Debug.Assert(request != null); if (request == null) throw new ArgumentNullException(nameof(request)); - return LoadObjects(pbfPath, cacheName, request, filter: null, stepLimit); + return LoadObjects(pbfPath, cacheName, request, filter: null, stepLimit, getAllRelations: !lowMemoryMode); } - public static OsmObjectResponse LoadObjects(string pbfPath, string cacheName, Func filter, int stepLimit = 0) + public static OsmObjectResponse LoadObjects( + string pbfPath, string cacheName, Func filter, int stepLimit = 0, bool lowMemoryMode = false) { Debug.Assert(filter != null); if (filter == null) throw new ArgumentNullException(nameof(filter)); - return LoadObjects(pbfPath, cacheName, request: null, filter, stepLimit); + return LoadObjects(pbfPath, cacheName, request: null, filter, stepLimit, getAllRelations: !lowMemoryMode); } private static OsmObjectResponse LoadObjects( - string pbfPath, string cacheName, OsmRequest request, Func filter, int stepLimit) + string pbfPath, string cacheName, OsmRequest request, Func filter, int stepLimit, bool getAllRelations) { Debug.Assert(pbfPath != null); Debug.Assert(!cacheName.IsNullOrWhiteSpace()); @@ -331,7 +359,7 @@ private static OsmObjectResponse LoadObjects( if (!FileClient.Exists(pbfPath)) throw new InvalidOperationException(); - LogService.BeginInfo("Load OSM objects"); + LogService.BeginInfo("Load OSM objects" + (getAllRelations ? string.Empty : " (low memory mode)")); var cacheStepPath = StepCachePath(cacheName, 1); if (FileClient.Exists(cacheStepPath)) @@ -341,13 +369,16 @@ private static OsmObjectResponse LoadObjects( LogService.LogInfo($"Step 1"); if (request != null) - context = Load(pbfPath, request); + context = Load(pbfPath, request, getAllRelations); else if (filter != null) - context = Load(pbfPath, filter); + context = Load(pbfPath, filter, getAllRelations); else throw new InvalidOperationException(); + if (getAllRelations) + FindRelations(context); + JsonFileClient.Write(cacheStepPath, context); } @@ -377,6 +408,47 @@ private static OsmObjectResponse LoadObjects( return objects; } + private static void FindRelations(OsmResponse context) + { + Debug.Assert(context.AllRelations != null); + + LogService.BeginInfo("Filter relations"); + var relations = context.Relations; + var missedRelationIds = new HashSet(context.MissedRelationIds); + var allRelation = context.AllRelations; + + void proceedRelationId(long relationId) + { + if (relations.ContainsKey(relationId) || missedRelationIds.Contains(relationId)) + return; + + if (allRelation.TryGetValue(relationId, out var relation)) + { + relations.Add(relation.Id, relation); + + var memberRelationIds = relation.MissedMembers.Where(i => i.Type == OsmGeoType.Relation) + .Select(i => i.Id); + + foreach (var memberRelationId in memberRelationIds) + proceedRelationId(memberRelationId); + } + else + missedRelationIds.Add(relationId); + } + + var deepMemberRelationIds = relations.Values.SelectMany(i => i.MissedMembers) + .Where(i => i.Type == OsmGeoType.Relation) + .Select(i => i.Id) + .Distinct() + .ToList(); + + foreach (var deepMemberRelationId in deepMemberRelationIds) + proceedRelationId(deepMemberRelationId); + + context.AllRelations = null; + LogService.EndInfo("Filter relations completed"); + } + private static bool LoadStep(string pbfPath, string cacheName, OsmResponse context, int step) { var cacheStepPath = StepCachePath(cacheName, step);