From ff72f11cc546064083efac4d9b6a32babbb2fbb5 Mon Sep 17 00:00:00 2001 From: Filip Strajnar Date: Mon, 24 Jun 2024 13:17:23 +0200 Subject: [PATCH] Asynchronously process OSM elements. --- OsmToDatabase.Peaks/Program.cs | 107 +++++++++++++++++---------------- 1 file changed, 56 insertions(+), 51 deletions(-) diff --git a/OsmToDatabase.Peaks/Program.cs b/OsmToDatabase.Peaks/Program.cs index 2fa1e3c..9c5abcf 100644 --- a/OsmToDatabase.Peaks/Program.cs +++ b/OsmToDatabase.Peaks/Program.cs @@ -7,62 +7,67 @@ long counter = 0; HashSet naturalPeakFeatures = new HashSet { "peak", "volcano" }; +async Task ProcessOsmElement(OsmGeo? element) +{ + if (element is Node node) + { + string? naturalValue = node.TagValueByKey("natural"); + + // Skip if natural tag is missing. + if (naturalValue is null) + { + return; + } + + // Skip if the node is not a desired feature. + if (!naturalPeakFeatures.Contains(naturalValue)) + { + return; + } + + // Skip if elevation is missing. + if (node.TagDoubleValueByKey("ele") is null) + { + return; + } + + Peak peak = new Peak + { + Id = node.Id.Value, + Version = node.Version.Value, + TimeStamp = node.TimeStamp.Value.ToUniversalTime(), + Latitude = node.Latitude.Value, + Longitude = node.Longitude.Value, + Elevation = node.TagDoubleValueByKey("ele"), + Prominence = node.TagDoubleValueByKey("prominence"), + Isolation = node.TagDoubleValueByKey("isolation"), + Name = node.TagValueByKey("name"), + AlternativeName = node.TagValueByKey("alt_name"), + EnglishName = node.TagValueByKey("name:en"), + EnglishAlternativeName = node.TagValueByKey("alt_name:en"), + ItalianName = node.TagValueByKey("name:it"), + SlovenianName = node.TagValueByKey("name:sl"), + GermanName = node.TagValueByKey("name:de"), + SummitRegisterPresent = node.TagValueByKey("summit:register") == "yes", + Wikidata = node.TagValueByKey("wikidata") + }; + await db.AddAsync(peak); + counter++; + + if ((counter % 50_000) == 0) + { + Console.WriteLine($"{counter}: Saving another batch."); + await db.SaveChangesAsync(); + } + } +} + using (var fileStream = new FileInfo(args[0]).OpenRead()) { var source = new PBFOsmStreamSource(fileStream); foreach (var element in source) { - if (element is Node node) - { - string? naturalValue = node.TagValueByKey("natural"); - - // Skip if natural tag is missing. - if (naturalValue is null) - { - continue; - } - - // Skip if the node is not a desired feature. - if (!naturalPeakFeatures.Contains(naturalValue)) - { - continue; - } - - // Skip if elevation is missing. - if (node.TagDoubleValueByKey("ele") is null) - { - continue; - } - - Peak peak = new Peak - { - Id = node.Id.Value, - Version = node.Version.Value, - TimeStamp = node.TimeStamp.Value.ToUniversalTime(), - Latitude = node.Latitude.Value, - Longitude = node.Longitude.Value, - Elevation = node.TagDoubleValueByKey("ele"), - Prominence = node.TagDoubleValueByKey("prominence"), - Isolation = node.TagDoubleValueByKey("isolation"), - Name = node.TagValueByKey("name"), - AlternativeName = node.TagValueByKey("alt_name"), - EnglishName = node.TagValueByKey("name:en"), - EnglishAlternativeName = node.TagValueByKey("alt_name:en"), - ItalianName = node.TagValueByKey("name:it"), - SlovenianName = node.TagValueByKey("name:sl"), - GermanName = node.TagValueByKey("name:de"), - SummitRegisterPresent = node.TagValueByKey("summit:register") == "yes", - Wikidata = node.TagValueByKey("wikidata") - }; - await db.AddAsync(peak); - counter++; - - if ((counter % 50_000) == 0) - { - Console.WriteLine($"{counter}: Saving another batch."); - await db.SaveChangesAsync(); - } - } + await ProcessOsmElement(element); } await db.SaveChangesAsync(); }