Skip to content

Commit

Permalink
Move to NE 5.1.2 - Add Config for new POVs (#2078)
Browse files Browse the repository at this point in the history
* split lookup depending on whether it's a country (#2075)

* return fclass_iso, fclass_tlc, and label_x, label_y for OSM features with matching NE features (#2082)

* adding TLC POV to NE export, YAML files and transform.py (#2081)

* adding tests

* upgrading to 5.1.2

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
  • Loading branch information
tgrigsby-sc and pre-commit-ci[bot] authored May 17, 2022
1 parent 66740a3 commit 24c26c6
Show file tree
Hide file tree
Showing 9 changed files with 239 additions and 58 deletions.
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,9 @@ test_server.port

# files generated during installation of software necessary for testing
.eggs/*

# build artifacts
build/*

# generated by jetbrains ides
.idea/*
70 changes: 38 additions & 32 deletions data/assets.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
bucket: nextzen-tile-assets
datestamp: 20220426
datestamp: 20220516

shapefiles:

Expand Down Expand Up @@ -55,98 +55,104 @@ shapefiles:
url: http://s3.amazonaws.com/tilezen-assets/curated/admin_areas_20180409.zip

- name: ne_110m_lakes
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/physical/ne_110m_lakes.zip
url: https://naciscdn.org/naturalearth/5.1.2/110m/physical/ne_110m_lakes.zip

- name: ne_50m_lakes
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/physical/ne_50m_lakes.zip
url: https://naciscdn.org/naturalearth/5.1.2/50m/physical/ne_50m_lakes.zip

- name: ne_10m_lakes
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_lakes.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/physical/ne_10m_lakes.zip

- name: ne_110m_ocean
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/physical/ne_110m_ocean.zip
url: https://naciscdn.org/naturalearth/5.1.2/110m/physical/ne_110m_ocean.zip

- name: ne_50m_ocean
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/physical/ne_50m_ocean.zip
url: https://naciscdn.org/naturalearth/5.1.2/50m/physical/ne_50m_ocean.zip

- name: ne_10m_ocean
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_ocean.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/physical/ne_10m_ocean.zip

- name: ne_50m_playas
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/physical/ne_50m_playas.zip
url: https://naciscdn.org/naturalearth/5.1.2/50m/physical/ne_50m_playas.zip

- name: ne_10m_playas
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_playas.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/physical/ne_10m_playas.zip

- name: ne_50m_urban_areas
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_urban_areas.zip
url: https://naciscdn.org/naturalearth/5.1.2/50m/cultural/ne_50m_urban_areas.zip

- name: ne_10m_urban_areas
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_urban_areas.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_urban_areas.zip

- name: ne_110m_land
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/physical/ne_110m_land.zip
url: https://naciscdn.org/naturalearth/5.1.2/110m/physical/ne_110m_land.zip

- name: ne_50m_land
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/physical/ne_50m_land.zip
url: https://naciscdn.org/naturalearth/5.1.2/50m/physical/ne_50m_land.zip

- name: ne_10m_land
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_land.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/physical/ne_10m_land.zip
tile: true

- name: ne_10m_populated_places
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_populated_places.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_populated_places.zip

- name: ne_110m_admin_0_boundary_lines_land
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/cultural/ne_110m_admin_0_boundary_lines_land.zip
url: https://naciscdn.org/naturalearth/5.1.2/110m/cultural/ne_110m_admin_0_boundary_lines_land.zip

- name: ne_50m_admin_0_boundary_lines_land
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_0_boundary_lines_land.zip
url: https://naciscdn.org/naturalearth/5.1.2/50m/cultural/ne_50m_admin_0_boundary_lines_land.zip

- name: ne_10m_admin_0_boundary_lines_land
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_boundary_lines_land.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_boundary_lines_land.zip

- name: ne_10m_admin_0_boundary_lines_map_units
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_boundary_lines_map_units.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_boundary_lines_map_units.zip

- name: ne_50m_admin_1_states_provinces_lines
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_1_states_provinces_lines.zip
url: https://naciscdn.org/naturalearth/5.1.2/50m/cultural/ne_50m_admin_1_states_provinces_lines.zip

- name: ne_10m_admin_1_states_provinces_lines
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_1_states_provinces_lines.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_1_states_provinces_lines.zip

- name: ne_50m_admin_0_boundary_lines_disputed_areas
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_0_boundary_lines_disputed_areas.zip
url: https://naciscdn.org/naturalearth/5.1.2/50m/cultural/ne_50m_admin_0_boundary_lines_disputed_areas.zip

- name: ne_50m_admin_0_boundary_lines_maritime_indicator_chn
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_0_boundary_lines_maritime_indicator_chn.zip
url: https://naciscdn.org/naturalearth/5.1.2/50m/cultural/ne_50m_admin_0_boundary_lines_maritime_indicator_chn.zip

- name: ne_10m_admin_0_boundary_lines_disputed_areas
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_boundary_lines_disputed_areas.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_boundary_lines_disputed_areas.zip

- name: ne_10m_admin_0_boundary_lines_maritime_indicator_chn
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_boundary_lines_maritime_indicator_chn.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_boundary_lines_maritime_indicator_chn.zip

- name: ne_10m_roads
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_roads.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_roads.zip

- name: ne_110m_coastline
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/physical/ne_110m_coastline.zip
url: https://naciscdn.org/naturalearth/5.1.2/110m/physical/ne_110m_coastline.zip

- name: ne_50m_coastline
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/physical/ne_50m_coastline.zip
url: https://naciscdn.org/naturalearth/5.1.2/50m/physical/ne_50m_coastline.zip

- name: ne_10m_coastline
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_coastline.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/physical/ne_10m_coastline.zip

- name: ne_10m_admin_0_countries
url: https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_countries.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_countries.zip

- name: ne_10m_admin_0_countries_iso
url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_countries_iso.zip

- name: ne_10m_admin_0_countries_tlc
url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_countries_tlc.zip

- name: ne_10m_admin_0_map_units
url: https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_map_units.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_map_units.zip

- name: ne_10m_admin_1_states_provinces
url: https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_1_states_provinces.zip
url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_1_states_provinces.zip

wikidata-queries:
- name: aerodrome_passenger_count
Expand Down
108 changes: 85 additions & 23 deletions data/functions.sql
Original file line number Diff line number Diff line change
Expand Up @@ -996,7 +996,7 @@ $$ LANGUAGE plpgsql IMMUTABLE;
-- returns a JSONB object containing __ne_min_zoom and __ne_max_zoom set to the
-- label min and max zoom of any matching row from the Natural Earth countries,
-- map units and states/provinces themes.
CREATE OR REPLACE FUNCTION tz_get_ne_min_max_zoom(wikidata_id TEXT)
CREATE OR REPLACE FUNCTION tz_get_ne_min_max_zoom(wikidata_id TEXT, place_tag TEXT)
RETURNS JSONB AS $$
DECLARE
min_zoom REAL;
Expand All @@ -1006,34 +1006,94 @@ BEGIN
RETURN '{}'::jsonb;
END IF;

-- first, try the countries table
SELECT
min_label, max_label INTO min_zoom, max_zoom
FROM ne_10m_admin_0_countries c
WHERE c.wikidataid = wikidata_id;
-- if it's a country, only look it up in the iso and tlc tables
IF place_tag='country' OR place_tag='unrecognized' THEN
SELECT
min_label, max_label INTO min_zoom, max_zoom
FROM ne_10m_admin_0_countries_iso i
WHERE i.wikidataid = wikidata_id;

-- if that fails, try map_units (which contains some sub-country but super-
-- state level stuff such as England, Scotland and Wales).
IF NOT FOUND THEN
IF NOT FOUND THEN
SELECT
min_label, max_label INTO min_zoom, max_zoom
FROM ne_10m_admin_0_countries_tlc t
WHERE t.wikidataid = wikidata_id;
END IF;
ELSE
-- try states and provinces if it's not a country
SELECT
min_label, max_label INTO min_zoom, max_zoom
FROM ne_10m_admin_0_map_units mu
WHERE mu.wikidataid = wikidata_id;
END IF;
min_label, max_label INTO min_zoom, max_zoom
FROM ne_10m_admin_1_states_provinces sp
WHERE sp.wikidataid = wikidata_id;

-- try states and provinces
-- finally, try localities
-- There is no concept of max_zoom for ne_10m_populated_places
IF NOT FOUND THEN
SELECT
pp.min_zoom, NULL INTO min_zoom, max_zoom
FROM ne_10m_populated_places pp
WHERE pp.wikidataid = wikidata_id;
END IF;
END IF;
-- return an empty JSONB rather than null, so that it can be safely
-- concatenated with whatever other JSONB rather than needing a check for
-- null.
IF NOT FOUND THEN
RETURN '{}'::jsonb;
END IF;
RETURN jsonb_build_object(
'__ne_min_zoom', min_zoom,
'__ne_max_zoom', max_zoom
);
END;
$$ LANGUAGE plpgsql STABLE;

CREATE OR REPLACE FUNCTION tz_get_fclass_and_label_position(wikidata_id TEXT, place_tag TEXT)
RETURNS JSONB AS $$
DECLARE
fclass_iso_var TEXT;
fclass_tlc_var TEXT;
label_x_var REAL;
label_y_var REAL;
BEGIN
IF wikidata_id IS NULL THEN
RETURN '{}'::jsonb;
END IF;

-- if it's a country, only look it up in the iso and tlc tables
IF place_tag='country' OR place_tag='unrecognized' THEN
SELECT
min_label, max_label INTO min_zoom, max_zoom
FROM ne_10m_admin_1_states_provinces sp
WHERE sp.wikidataid = wikidata_id;
i.fclass_iso, i.fclass_tlc, i.label_x, i.label_y INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var
FROM ne_10m_admin_0_countries_iso i
WHERE i.wikidataid = wikidata_id;

IF NOT FOUND THEN
SELECT
t.fclass_iso, t.fclass_tlc, t.label_x, t.label_y INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var
FROM ne_10m_admin_0_countries_tlc t
WHERE t.wikidataid = wikidata_id;
END IF;

IF NOT FOUND THEN
RETURN '{}'::jsonb;
END IF;
RETURN jsonb_build_object(
'__ne_fclass_iso', fclass_iso_var,
'__ne_fclass_tlc', fclass_tlc_var,
'__ne_label_x', label_x_var,
'__ne_label_y', label_y_var
);
END IF;

SELECT
sp.fclass_iso, sp.fclass_tlc, sp.longitude, sp.latitude INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var
FROM ne_10m_admin_1_states_provinces sp
WHERE sp.wikidataid = wikidata_id;

-- finally, try localities
-- There is no concept of max_zoom for ne_10m_populated_places
IF NOT FOUND THEN
SELECT
pp.min_zoom, NULL INTO min_zoom, max_zoom
SELECT
pp.fclass_iso, pp.fclass_tlc, pp.longitude, pp.latitude INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var
FROM ne_10m_populated_places pp
WHERE pp.wikidataid = wikidata_id;
END IF;
Expand All @@ -1045,9 +1105,11 @@ BEGIN
RETURN '{}'::jsonb;
END IF;
RETURN jsonb_build_object(
'__ne_min_zoom', min_zoom,
'__ne_max_zoom', max_zoom
);
'__ne_fclass_iso', fclass_iso_var,
'__ne_fclass_tlc', fclass_tlc_var,
'__ne_label_x', label_x_var,
'__ne_label_y', label_y_var
);
END;
$$ LANGUAGE plpgsql STABLE;

Expand Down
102 changes: 102 additions & 0 deletions integration-test/2081-tlc-pov.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
import dsl

from . import FixtureTest


class TestTLCPOV(FixtureTest):
def test_tlc_ne_place(self):
import dsl

z, x, y = 16, 0, 0

self.generate_fixtures(
dsl.way(1, dsl.tile_centre_shape(z, x, y), {
'name': 'Foo',
'featurecla': 'Admin-0 capital',
'fclass_iso': 'Admin-1 capital',
'fclass_tlc': 'Admin-0 capital',
'scalerank': 4,
'min_zoom': 4,
'source': 'naturalearthdata.com',
}),
)

self.assert_has_feature(
z, x, y, 'places', {
'kind': 'locality',
'country_capital': type(True),
'country_capital:iso': type(False),
'region_capital:iso': type(True),
'country_capital:tlc': type(True),
})

def test_tlc_ne_boundary(self):
import dsl

z, x, y = 16, 0, 0

self.generate_fixtures(
dsl.way(1, dsl.tile_diagonal(z, x, y), {
'name': 'Foo',
'featurecla': 'Admin-1 region boundary',
'fclass_iso': 'Admin-1 region boundary',
'fclass_tlc': 'International boundary (verify)',
'scalerank': 4,
'min_zoom': 4,
'source': 'naturalearthdata.com',
}),
)

self.assert_has_feature(
z, x, y, 'boundaries', {
'kind': 'macroregion',
'kind:iso': 'macroregion',
'kind:tlc': 'country',
})

def test_osm_admin_level_viewpoint_tlc(self):
z, x, y = (16, 39109, 26572)

self.generate_fixtures(
dsl.way(726514231, dsl.tile_diagonal(z, x, y), {
'admin_level': '4',
'admin_level:ISO': '8',
'admin_level:TLC': '8',
'boundary': 'disputed',
'name': 'Viewpoints on Disputed Administrative Boundaries',
'type': 'linestring',
'source': 'openstreetmap.org',
}),
)

self.assert_has_feature(
z, x, y, 'boundaries', {
'id': 726514231,
'kind': 'disputed_reference_line',
'kind:iso': 'locality',
'kind:tlc': 'locality',
})

def test_osm_places_with_viewpoint_tlc(self):
import dsl

z, x, y = (10, 856, 441)

self.generate_fixtures(
dsl.point(432425099, (120.9820179, 23.9739374), {
'name': 'Test place',
'place': 'country',
'place:ISO': 'state',
'place:TLC': 'district',
'source': 'openstreetmap.org',
'source:sqkm': 'CIA World Factbook',
}),
)

self.assert_has_feature(
z, x, y, 'places', {
'id': 432425099,
'kind': 'country',
'kind:iso': 'region',
'kind:tlc': 'county'
})
Loading

0 comments on commit 24c26c6

Please sign in to comment.