Skip to content

Commit

Permalink
Get all relations mode
Browse files Browse the repository at this point in the history
  • Loading branch information
@chubrik
chubrik committed May 18, 2019
1 parent 18f9b05 commit e88ddb9
Show file tree
Hide file tree
Showing 3 changed files with 94 additions and 20 deletions.
0 OsmDataKit/Extensions/GeoCoordsExtensions.cs → OsmDataKit/Extensions/GeoPointExtensions.cs
View file
File renamed without changes.
2 changes: 2 additions & 0 deletions OsmDataKit/Models/OsmResponse.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,5 +40,7 @@ public RelationObject[] _relations
get => Relations.Values.ToArray();
set => Relations = value.ToDictionary(i => i.Id);
}

internal Dictionary<long, RelationObject> AllRelations { get; set; }
}
}
112 changes: 92 additions & 20 deletions OsmDataKit/OsmService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,10 @@ public static void ValidateSource(string pbfPath)

#region Load

public static OsmResponse Load(string pbfPath, Func<OsmGeo, bool> filter)
public static OsmResponse Load(string pbfPath, Func<OsmGeo, bool> filter) =>
Load(pbfPath, filter, getAllRelations: false);

private static OsmResponse Load(string pbfPath, Func<OsmGeo, bool> filter, bool getAllRelations)
{
Debug.Assert(pbfPath != null);
Debug.Assert(filter != null);
Expand All @@ -85,24 +88,38 @@ public static OsmResponse Load(string pbfPath, Func<OsmGeo, bool> filter)
var foundNodes = new Dictionary<long, NodeObject>();
var foundWays = new Dictionary<long, WayObject>();
var foundRelations = new Dictionary<long, RelationObject>();
var allRelations = getAllRelations ? new Dictionary<long, RelationObject>() : null;

using (var fileStream = FileClient.OpenRead(pbfPath))
{
var source = new PBFOsmStreamSource(fileStream);

foreach (var osmGeo in source.Where(filter))
foreach (var osmGeo in source)
switch (osmGeo.Type)
{
case OsmGeoType.Node:
foundNodes.Add(osmGeo.Id.GetValueOrDefault(), new NodeObject(osmGeo as Node));

if (filter(osmGeo))
foundNodes.Add(osmGeo.Id.GetValueOrDefault(), new NodeObject(osmGeo as Node));

break;

case OsmGeoType.Way:
foundWays.Add(osmGeo.Id.GetValueOrDefault(), new WayObject(osmGeo as Way));

if (filter(osmGeo))
foundWays.Add(osmGeo.Id.GetValueOrDefault(), new WayObject(osmGeo as Way));

break;

case OsmGeoType.Relation:
foundRelations.Add(osmGeo.Id.GetValueOrDefault(), new RelationObject(osmGeo as Relation));
RelationObject relation = null;

if (getAllRelations)
allRelations.Add(osmGeo.Id.GetValueOrDefault(), relation = new RelationObject(osmGeo as Relation));

if (filter(osmGeo))
foundRelations.Add(osmGeo.Id.GetValueOrDefault(), relation ?? new RelationObject(osmGeo as Relation));

break;

default:
Expand All @@ -112,10 +129,13 @@ public static OsmResponse Load(string pbfPath, Func<OsmGeo, bool> filter)

LogService.LogInfo($"Loaded: {foundNodes.Count} nodes, {foundWays.Count} ways, {foundRelations.Count} relations");
LogService.EndInfo("Load OSM data completed");
return new OsmResponse { Nodes = foundNodes, Ways = foundWays, Relations = foundRelations };
return new OsmResponse { Nodes = foundNodes, Ways = foundWays, Relations = foundRelations, AllRelations = allRelations };
}

public static OsmResponse Load(string pbfPath, OsmRequest request)
public static OsmResponse Load(string pbfPath, OsmRequest request) =>
Load(pbfPath, request, getAllRelations: false);

private static OsmResponse Load(string pbfPath, OsmRequest request, bool getAllRelations)
{
Debug.Assert(pbfPath != null);
Debug.Assert(request != null);
Expand Down Expand Up @@ -143,6 +163,7 @@ public static OsmResponse Load(string pbfPath, OsmRequest request)
var foundNodes = new Dictionary<long, NodeObject>();
var foundWays = new Dictionary<long, WayObject>();
var foundRelations = new Dictionary<long, RelationObject>();
var allRelations = getAllRelations ? new Dictionary<long, RelationObject>() : null;
List<long> missedNodeIds = null;
List<long> missedWayIds = null;
List<long> missedRelationIds = null;
Expand Down Expand Up @@ -190,7 +211,7 @@ public static OsmResponse Load(string pbfPath, OsmRequest request)

LogService.LogInfo("Loaded 0 ways");

if (requestRelationIds.Count > 0)
if (requestRelationIds.Count > 0 || getAllRelations)
{
thisType = OsmGeoType.Relation;
continue;
Expand Down Expand Up @@ -226,7 +247,7 @@ public static OsmResponse Load(string pbfPath, OsmRequest request)
else
LogService.LogWarning($"{logMessage} ({missedWayIds.Count} missed)");

if (requestRelationIds.Count > 0)
if (requestRelationIds.Count > 0 || getAllRelations)
{
thisType = OsmGeoType.Relation;
continue;
Expand All @@ -240,12 +261,16 @@ public static OsmResponse Load(string pbfPath, OsmRequest request)
continue;

id = osmGeo.Id.GetValueOrDefault();
RelationObject relation = null;

if (getAllRelations)
allRelations.Add(id, relation = new RelationObject(osmGeo as Relation));

if (requestRelationIds.Contains(id))
{
foundRelations.Add(id, new RelationObject(osmGeo as Relation));
foundRelations.Add(id, relation ?? new RelationObject(osmGeo as Relation));

if (foundRelations.Count == requestRelationIds.Count)
if (!getAllRelations && foundRelations.Count == requestRelationIds.Count)
goto Complete;
}

Expand Down Expand Up @@ -275,36 +300,39 @@ public static OsmResponse Load(string pbfPath, OsmRequest request)
Relations = foundRelations,
MissedNodeIds = missedNodeIds ?? new List<long>(0),
MissedWayIds = missedWayIds ?? new List<long>(0),
MissedRelationIds = missedRelationIds ?? new List<long>(0)
MissedRelationIds = missedRelationIds ?? new List<long>(0),
AllRelations = allRelations
};
}

#endregion

#region Load objects

public static OsmObjectResponse LoadObjects(string pbfPath, string cacheName, OsmRequest request, int stepLimit = 0)
public static OsmObjectResponse LoadObjects(
string pbfPath, string cacheName, OsmRequest request, int stepLimit = 0, bool lowMemoryMode = false)
{
Debug.Assert(request != null);

if (request == null)
throw new ArgumentNullException(nameof(request));

return LoadObjects(pbfPath, cacheName, request, filter: null, stepLimit);
return LoadObjects(pbfPath, cacheName, request, filter: null, stepLimit, getAllRelations: !lowMemoryMode);
}

public static OsmObjectResponse LoadObjects(string pbfPath, string cacheName, Func<OsmGeo, bool> filter, int stepLimit = 0)
public static OsmObjectResponse LoadObjects(
string pbfPath, string cacheName, Func<OsmGeo, bool> filter, int stepLimit = 0, bool lowMemoryMode = false)
{
Debug.Assert(filter != null);

if (filter == null)
throw new ArgumentNullException(nameof(filter));

return LoadObjects(pbfPath, cacheName, request: null, filter, stepLimit);
return LoadObjects(pbfPath, cacheName, request: null, filter, stepLimit, getAllRelations: !lowMemoryMode);
}

private static OsmObjectResponse LoadObjects(
string pbfPath, string cacheName, OsmRequest request, Func<OsmGeo, bool> filter, int stepLimit)
string pbfPath, string cacheName, OsmRequest request, Func<OsmGeo, bool> filter, int stepLimit, bool getAllRelations)
{
Debug.Assert(pbfPath != null);
Debug.Assert(!cacheName.IsNullOrWhiteSpace());
Expand All @@ -331,7 +359,7 @@ private static OsmObjectResponse LoadObjects(
if (!FileClient.Exists(pbfPath))
throw new InvalidOperationException();

LogService.BeginInfo("Load OSM objects");
LogService.BeginInfo("Load OSM objects" + (getAllRelations ? string.Empty : " (low memory mode)"));
var cacheStepPath = StepCachePath(cacheName, 1);

if (FileClient.Exists(cacheStepPath))
Expand All @@ -341,13 +369,16 @@ private static OsmObjectResponse LoadObjects(
LogService.LogInfo($"Step 1");

if (request != null)
context = Load(pbfPath, request);
context = Load(pbfPath, request, getAllRelations);
else
if (filter != null)
context = Load(pbfPath, filter);
context = Load(pbfPath, filter, getAllRelations);
else
throw new InvalidOperationException();

if (getAllRelations)
FindRelations(context);

JsonFileClient.Write(cacheStepPath, context);
}

Expand Down Expand Up @@ -377,6 +408,47 @@ private static OsmObjectResponse LoadObjects(
return objects;
}

private static void FindRelations(OsmResponse context)
{
Debug.Assert(context.AllRelations != null);

LogService.BeginInfo("Filter relations");
var relations = context.Relations;
var missedRelationIds = new HashSet<long>(context.MissedRelationIds);
var allRelation = context.AllRelations;

void proceedRelationId(long relationId)
{
if (relations.ContainsKey(relationId) || missedRelationIds.Contains(relationId))
return;

if (allRelation.TryGetValue(relationId, out var relation))
{
relations.Add(relation.Id, relation);

var memberRelationIds = relation.MissedMembers.Where(i => i.Type == OsmGeoType.Relation)
.Select(i => i.Id);

foreach (var memberRelationId in memberRelationIds)
proceedRelationId(memberRelationId);
}
else
missedRelationIds.Add(relationId);
}

var deepMemberRelationIds = relations.Values.SelectMany(i => i.MissedMembers)
.Where(i => i.Type == OsmGeoType.Relation)
.Select(i => i.Id)
.Distinct()
.ToList();

foreach (var deepMemberRelationId in deepMemberRelationIds)
proceedRelationId(deepMemberRelationId);

context.AllRelations = null;
LogService.EndInfo("Filter relations completed");
}

private static bool LoadStep(string pbfPath, string cacheName, OsmResponse context, int step)
{
var cacheStepPath = StepCachePath(cacheName, step);
Expand Down

0 comments on commit e88ddb9

Please sign in to comment.