diff --git a/entrypoint.sh b/entrypoint.sh index 15184b6..4b76b1c 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -7,12 +7,21 @@ if [[ $# -lt 2 ]]; then exit 1 fi +TEST=0 +for arg in "$@"; do + case $arg in + --test) TEST=1 ;; + esac +done + echo "Extracting feature layers from $1" ./process.sh "$@" -for input_file in "$2"/*.parquet; do - echo "Sorting and compressing $input_file" - output_file="$2/$(basename -s .parquet $input_file)-optimized.parquet" - ./postprocess.sh $input_file $output_file - mv $output_file $input_file -done +if [ "$TEST" = "0" ]; then + for input_file in "$2"/*.parquet; do + echo "Sorting and compressing $input_file" + output_file="$2/$(basename -s .parquet $input_file)-optimized.parquet" + ./postprocess.sh $input_file $output_file + mv $output_file $input_file + done +fi \ No newline at end of file diff --git a/process.sh b/process.sh index 7f1290f..e942fc3 100755 --- a/process.sh +++ b/process.sh @@ -4,7 +4,7 @@ # using DuckDB with the osmium and spatial extensions. # # Usage: process.sh [--buildings] [--highways] ... -# [--osmium-index-type=TYPE] [--duckdb-memory-limit=LIMIT] +# [--test] [--osmium-index-type=TYPE] [--duckdb-memory-limit=LIMIT] # # --osmium-index-type sets osmium's node location index type. The default # is 'flex_mem' which works well for both small and large extracts. For @@ -27,8 +27,9 @@ OUTPUT_DIR="$2" shift 2 # Parse optional layer flags -BUILDINGS=0; HIGHWAYS=0; BOUNDARIES=0; SETTLEMENTS=0; PARKS=0 +BUILDINGS=0; HIGHWAYS=0; BOUNDARIES=0; SETTLEMENTS=0; PARKS=0; WATER=0 ALL=1 +TEST=0 OSMIUM_INDEX_TYPE="" DUCKDB_MEMORY_LIMIT="" @@ -39,6 +40,8 @@ for arg in "$@"; do --boundaries) BOUNDARIES=1; ALL=0 ;; --settlements) SETTLEMENTS=1; ALL=0 ;; --parks) PARKS=1; ALL=0 ;; + --water) WATER=1; ALL=0 ;; + --test) TEST=1 ;; --osmium-index-type=*) OSMIUM_INDEX_TYPE="${arg#*=}" ;; --duckdb-memory-limit=*) DUCKDB_MEMORY_LIMIT="${arg#*=}" ;; *) echo "Unknown argument: $arg" >&2; exit 1 ;; @@ -46,7 +49,7 @@ for arg in "$@"; do done if [ "$ALL" = "1" ]; then - BUILDINGS=1; HIGHWAYS=1; BOUNDARIES=1; SETTLEMENTS=1; PARKS=1 + BUILDINGS=1; HIGHWAYS=1; BOUNDARIES=1; SETTLEMENTS=1; PARKS=1; WATER=1 fi mkdir -p "$OUTPUT_DIR" @@ -56,15 +59,28 @@ mkdir -p "$OUTPUT_DIR" run_layer() { local name="$1" local output="${OUTPUT_DIR}/${name}.parquet" - echo "Extracting ${name} layer" - { - cat "${SCRIPT_DIR}/sql/macros.sql" - [ -n "$DUCKDB_MEMORY_LIMIT" ] && echo "SET memory_limit = '${DUCKDB_MEMORY_LIMIT}';" - [ -n "$OSMIUM_INDEX_TYPE" ] && echo "SET osmium_index_type = '${OSMIUM_INDEX_TYPE}';" - cat "${SCRIPT_DIR}/sql/${name}.sql" - } | \ - sed "s|{{INPUT}}|${INPUT}|g; s|{{OUTPUT}}|${output}|g" | \ - duckdb --unsigned + local script_file; + + if [ "$TEST" = "1" ]; then + local script_file="${SCRIPT_DIR}/sql/${name}_test.sql" + if [ -f "$script_file" ]; then + echo "Testing ${name} layer" + fi + else + echo "Extracting ${name} layer" + local script_file="${SCRIPT_DIR}/sql/${name}.sql" + fi + + if [ -f "$script_file" ]; then + { + cat "${SCRIPT_DIR}/sql/macros.sql" + [ -n "$DUCKDB_MEMORY_LIMIT" ] && echo "SET memory_limit = '${DUCKDB_MEMORY_LIMIT}';" + [ -n "$OSMIUM_INDEX_TYPE" ] && echo "SET osmium_index_type = '${OSMIUM_INDEX_TYPE}';" + cat "$script_file" + } | \ + sed "s|{{INPUT}}|${INPUT}|g; s|{{OUTPUT}}|${output}|g" | \ + duckdb --unsigned + fi } [ "$BUILDINGS" = "1" ] && run_layer buildings @@ -72,5 +88,6 @@ run_layer() { [ "$BOUNDARIES" = "1" ] && run_layer boundaries [ "$SETTLEMENTS" = "1" ] && run_layer settlements [ "$PARKS" = "1" ] && run_layer parks +[ "$WATER" = "1" ] && run_layer water echo "Done" diff --git a/sql/macros.sql b/sql/macros.sql index 01f9021..42d2f55 100644 --- a/sql/macros.sql +++ b/sql/macros.sql @@ -32,3 +32,47 @@ CREATE OR REPLACE MACRO prefix_map_split(pfx, t) AS ( ) ) ); + +CREATE OR REPLACE MACRO assert_col_not_empty(table_name, col_name) AS ( + SELECT CASE + WHEN NOT EXISTS ( + -- Must rewrap the col_name in an extra layer of quoting. + SELECT 1 FROM query('SELECT 1 FROM ' || table_name || ' WHERE "' || col_name || '" IS NOT NULL LIMIT 1') + ) + THEN CAST(error('Assertion Failed: Empty column: ' || col_name) AS INTEGER) + ELSE 1 + END +); + +CREATE OR REPLACE MACRO assert_map_not_empty(table_name, col_name) AS ( + SELECT CASE + WHEN NOT EXISTS ( + -- Must rewrap the col_name in an extra layer of quoting. + SELECT 1 FROM query('SELECT 1 FROM ' || table_name || ' WHERE cardinality("' || col_name || '") > 0 LIMIT 1') + ) + THEN CAST(error('Assertion Failed: Empty map found for column: ' || col_name) AS INTEGER) + ELSE 1 + END +); + +CREATE OR REPLACE MACRO assert_tag_not_empty(table_name, k, v) AS ( + SELECT CASE + WHEN NOT EXISTS ( + SELECT 1 FROM query( + printf('SELECT 1 FROM %s WHERE "%s" = ''%s'' LIMIT 1',table_name, k, v) + ) + ) + THEN CAST(error(printf('Assertion Failed: Empty tag for "%s"=''%s''', k, v)) AS INTEGER) + ELSE 1 + END +); + +CREATE OR REPLACE MACRO assert_stmt_not_empty(stmt) AS ( + SELECT CASE + WHEN NOT EXISTS ( + SELECT 1 FROM query(stmt) + ) + THEN CAST(error(printf('Assertion Failed: stmt ''%s'' produced no values', stmt)) AS INTEGER) + ELSE 1 + END +); diff --git a/sql/water.sql b/sql/water.sql new file mode 100644 index 0000000..2f5ea85 --- /dev/null +++ b/sql/water.sql @@ -0,0 +1,151 @@ +CREATE OR REPLACE TEMP TABLE bridges_unfiltered AS +SELECT type, id, tags, geometry +FROM '{{INPUT}}' +WHERE tags['man_made'] = 'bridge' OR tags['bridge'] IS NOT NULL; + +CREATE OR REPLACE TEMP TABLE water_features AS +SELECT type, id, tags, geometry +FROM '{{INPUT}}' +WHERE ( + (kind = 'line' AND tags['waterway'] IS NOT NULL) OR + (kind = 'area' AND ( + tags['natural'] IN ('water', 'coastline', 'wetland') OR + tags['landuse'] IN ('basin', 'reservoir', 'harbour') OR + tags['waterway'] IS NOT NULL + )) OR + tags['man_made'] IN ('pier', 'breakwater', 'groyne', 'lighthouse', 'beacon', 'buoy', 'offshore_platform', 'pumping_station', 'water_well', 'spring') OR + ( + tags['man_made'] = 'monitoring_station' AND ( + tags['monitoring:water'] IS NOT NULL OR + tags['monitoring:water_level'] IS NOT NULL OR + tags['monitoring:water_quality'] IS NOT NULL + ) + ) OR + tags['historic'] IN ('wreck','ship', 'aquaduct') OR + tags['seamark:type'] IS NOT NULL OR + tags['route'] IN ('ferry', 'portage') OR + tags['leisure'] IN ('slipway', 'marina', 'swimming_pool', 'swimming_area', 'water_park') OR + tags['amenity'] IN ('drinking_water', 'foot_shower', 'shower', 'boat_rental') OR + tags['sport'] IN ('canoe', 'cliff_diving', 'diving', 'dragon_boat', 'rowing', 'sailing', 'scuba_diving', 'surfing', 'swimming', 'wakeboarding', 'water_ski', 'windsurfing') OR + tags['portage'] IS NOT NULL OR + tags['canoe'] IS NOT NULL OR + tags['canoe_rental'] IS NOT NULL OR + tags['mooring'] IS NOT NULL OR + (tags['landuse'] = 'industrial' AND tags['industrial'] = 'port') OR + ( -- All deprecated in favor of tags['emergency'] = 'water_rescue' + tags['emergency'] IN ('lifeboat_station', 'marine_rescue') OR + tags['amenity'] = 'lifeboat_station' + ) OR + tags['emergency'] IN ( 'lifeguard', 'water_rescue', 'life_ring', 'throw_bag', 'rescue_buoy') OR + ( + tags['emergency'] = 'assembly_point' AND ( + -- Unfortunately not all tsunami assembly points have the correct assembly_point:tsunami tag + -- https://www.openstreetmap.org/node/4368193931 + tags['assembly_point:tsunami'] IS NOT NULL OR + tags['assembly_point:storm_surge'] IS NOT NULL + ) + ) OR + tags['ford'] IS NOT NULL OR + tags['tidal'] IS NOT NULL OR + tags['flood_prone'] IS NOT NULL OR + tags['whitewater'] IS NOT NULL OR + tags['club'] in ('sailing', 'scuba_diving', 'surf_life_saving') OR + tags['shop'] = 'boat' OR + tags['boat:type'] IS NOT NULL +); + +-- Known to exclude https://www.openstreetmap.org/way/35457618 from the Oregon region +CREATE OR REPLACE TEMP TABLE water_bridges AS +SELECT b.type, b.id, b.tags, b.geometry +FROM bridges_unfiltered b +JOIN water_features w + ON b.geometry && w.geometry + WHERE ST_Intersects(b.geometry, w.geometry); + +COPY ( + WITH raw AS ( + -- We call `SELECT DISTINCT` here instead of when the building of the "water_bridges" table + -- to work around a floating point exception which randomly goes away if you `PRAGMA threads = 1` + SELECT DISTINCT type, id, tags, geometry FROM water_bridges + + UNION ALL + SELECT type, id, tags, geometry FROM water_features + ) + SELECT + type, + id, + tags['natural'] AS "natural", + tags['waterway'] AS waterway, + tags['man_made'] AS man_made, + tags['historic'] AS historic, + tags['route'] AS route, + tags['intermittent'] AS intermittent, + tags['tunnel'] AS tunnel, + tags['covered'] AS covered, + tags['salt'] AS salt, + tags['boat'] AS boat, + tags['motorboat'] AS motorboat, + tags['canoe'] AS canoe, + tags['highway'] AS highway, + tags['portage'] AS portage, + tags['mooring'] AS mooring, + tags['seasonal'] AS seasonal, + tags['water'] AS water, + tags['bridge'] AS bridge, + tags['lifeguard'] AS lifeguard, + tags['emergency'] AS emergency, + tags['landuse'] AS landuse, + tags['industrial'] AS industrial, + tags['amenity'] AS amenity, + tags['leisure'] AS leisure, + tags['access'] AS access, + tags['fee'] AS fee, + tags['surface'] AS surface, + tags['ford'] AS ford, + tags['tidal'] AS tidal, + tags['flood_prone'] AS flood_prone, + tags['sport'] AS sport, + tags['wheelchair'] AS wheelchair, + tags['club'] AS club, + tags['whitewater'] AS whitewater, + tags['shop'] AS shop, + tags['canoe_rental'] AS canoe_rental, + tags['boat'] AS boat, + tags['ship'] AS ship, + tags['pump'] AS pump, -- For man_made=water_well + tags['drinking_water'] AS drinking_water, -- For man_made=water_well + tags['handle'] AS handle, -- For man_made=water_well + tags['mechanical_driver'] AS mechanical_driver, -- For man_made=water_well + tags['depth'] AS depth, -- For man_made=water_well + tags['mechanism'] AS mechanism, -- For man_made=water_well + prefix_map('pump:', tags) AS "pump:", -- For man_made=water_well + prefix_map('seamark:', tags) AS "seamark:", + prefix_map('assembly_point:', tags) AS "assembly_point:", + prefix_map('monitoring:', tags) AS "monitoring:", + prefix_map('whitewater:', tags) AS "whitewater:", + prefix_map('addr:', tags) AS "addr:", + prefix_map_split('boat:', tags) AS "boat:", + split_multi(tags['name']) AS name, + prefix_map_split('name:', tags) AS names, + split_multi(tags['official_name']) AS official_name, + prefix_map_split('official_name:', tags) AS official_names, + split_multi(tags['old_name']) AS old_name, + prefix_map_split('old_name:', tags) AS old_names, + split_multi(tags['alt_name']) AS alt_name, + prefix_map_split('alt_name:', tags) AS alt_names, + split_multi(tags['short_name']) AS short_name, + prefix_map_split('short_name:', tags) AS short_names, + tags['operator'] AS operator, + tags['description'] AS description, + tags['source'] AS source, + tags['wikidata'] AS wikidata, + tags['wikipedia'] AS wikipedia, + { + xmin: ST_XMin(geometry)::FLOAT, + ymin: ST_YMin(geometry)::FLOAT, + xmax: ST_XMax(geometry)::FLOAT, + ymax: ST_YMax(geometry)::FLOAT + } AS bbox, + geometry + FROM raw +) TO '{{OUTPUT}}' WITH (FORMAT PARQUET, COMPRESSION ZSTD); \ No newline at end of file diff --git a/sql/water_test.sql b/sql/water_test.sql new file mode 100644 index 0000000..f25c493 --- /dev/null +++ b/sql/water_test.sql @@ -0,0 +1,111 @@ + +-- This script unforunately depends on values being present in the region, it +-- has been known to succesfully pass with https://download.geofabrik.de/north-america/us/washington-latest.osm.pbf +.mode trash +CREATE VIEW water AS SELECT * FROM '{{OUTPUT}}'; +SELECT assert_col_not_empty('water', 'natural'); +SELECT assert_col_not_empty('water', 'waterway'); +SELECT assert_col_not_empty('water', 'man_made'); +SELECT assert_col_not_empty('water', 'historic'); +SELECT assert_col_not_empty('water', 'route'); +SELECT assert_col_not_empty('water', 'intermittent'); +SELECT assert_col_not_empty('water', 'tunnel'); +SELECT assert_col_not_empty('water', 'covered'); +SELECT assert_col_not_empty('water', 'salt'); +SELECT assert_col_not_empty('water', 'boat'); +SELECT assert_col_not_empty('water', 'motorboat'); +SELECT assert_col_not_empty('water', 'canoe'); +SELECT assert_col_not_empty('water', 'highway'); +SELECT assert_col_not_empty('water', 'portage'); +SELECT assert_col_not_empty('water', 'canoe'); +SELECT assert_col_not_empty('water', 'mooring'); +SELECT assert_col_not_empty('water', 'intermittent'); +SELECT assert_col_not_empty('water', 'seasonal'); +SELECT assert_col_not_empty('water', 'water'); +SELECT assert_col_not_empty('water', 'bridge'); +SELECT assert_col_not_empty('water', 'lifeguard'); +SELECT assert_col_not_empty('water', 'emergency'); +SELECT assert_col_not_empty('water', 'landuse'); +SELECT assert_col_not_empty('water', 'industrial'); +SELECT assert_col_not_empty('water', 'amenity'); +SELECT assert_col_not_empty('water', 'ford'); +SELECT assert_col_not_empty('water', 'tidal'); +SELECT assert_col_not_empty('water', 'flood_prone'); +SELECT assert_col_not_empty('water', 'wheelchair'); +SELECT assert_col_not_empty('water', 'whitewater'); +SELECT assert_col_not_empty('water', 'club'); +SELECT assert_col_not_empty('water', 'shop'); +SELECT assert_col_not_empty('water', 'boat'); +SELECT assert_col_not_empty('water', 'ship'); +SELECT assert_col_not_empty('water', 'canoe_rental'); + +SELECT assert_map_not_empty('water', 'seamark:'); +SELECT assert_map_not_empty('water', 'assembly_point:'); +SELECT assert_map_not_empty('water', 'monitoring:'); +SELECT assert_map_not_empty('water', 'addr:'); +SELECT assert_map_not_empty('water', 'boat:'); +-- Not present in the washington region +-- SELECT assert_map_not_empty('water', 'whitewater:'); + +SELECT assert_tag_not_empty('water', 'leisure','slipway'); +SELECT assert_tag_not_empty('water', 'lifeguard','tower'); +SELECT assert_tag_not_empty('water', 'emergency','lifeguard'); +SELECT assert_tag_not_empty('water', 'landuse','basin'); +SELECT assert_tag_not_empty('water', 'landuse','reservoir'); +SELECT assert_tag_not_empty('water', 'landuse','harbour'); +SELECT assert_tag_not_empty('water', 'route','portage'); +SELECT assert_tag_not_empty('water', 'route','ferry'); +SELECT assert_tag_not_empty('water', 'historic','wreck'); +SELECT assert_tag_not_empty('water', 'historic','ship'); +SELECT assert_tag_not_empty('water', 'man_made','pier'); +SELECT assert_tag_not_empty('water', 'man_made','breakwater'); +SELECT assert_tag_not_empty('water', 'man_made','groyne'); +SELECT assert_tag_not_empty('water', 'man_made','lighthouse'); +SELECT assert_tag_not_empty('water', 'man_made','beacon'); +SELECT assert_tag_not_empty('water', 'man_made','buoy'); +SELECT assert_tag_not_empty('water', 'man_made','pumping_station'); +SELECT assert_tag_not_empty('water', 'man_made','water_well'); +SELECT assert_tag_not_empty('water', 'man_made','bridge'); +SELECT assert_tag_not_empty('water', 'waterway','access_point'); +SELECT assert_tag_not_empty('water', 'shop','boat'); + +-- Amenities and leisure +SELECT assert_tag_not_empty('water', 'amenity','drinking_water'); +SELECT assert_tag_not_empty('water', 'amenity','foot_shower'); +SELECT assert_tag_not_empty('water', 'amenity','shower'); +SELECT assert_tag_not_empty('water', 'amenity','boat_rental'); +SELECT assert_tag_not_empty('water', 'leisure','swimming_area'); +SELECT assert_tag_not_empty('water', 'leisure','swimming_pool'); +SELECT assert_tag_not_empty('water', 'leisure','water_park'); +SELECT assert_tag_not_empty('water', 'sport', 'canoe'); +SELECT assert_tag_not_empty('water', 'sport', 'diving'); +SELECT assert_tag_not_empty('water', 'sport', 'rowing'); +SELECT assert_tag_not_empty('water', 'sport', 'sailing'); +SELECT assert_tag_not_empty('water', 'sport', 'scuba_diving'); +SELECT assert_tag_not_empty('water', 'sport', 'surfing'); +SELECT assert_tag_not_empty('water', 'sport', 'swimming'); +SELECT assert_tag_not_empty('water', 'sport', 'water_ski'); +SELECT assert_tag_not_empty('water', 'sport', 'windsurfing'); +-- Not present in the washington region +-- +-- SELECT assert_tag_not_empty('water', 'sport', 'cliff_diving'); +-- SELECT assert_tag_not_empty('water', 'sport', 'dragon_boat'); +-- SELECT assert_tag_not_empty('water', 'sport', 'wakeboarding'); + +SELECT assert_stmt_not_empty('SELECT 1 FROM water WHERE ("landuse" = ''industrial'' AND "industrial" = ''port'') LIMIT 1'); + +-- Not present in the washington region +-- SELECT assert_tag_not_empty('water', 'emergency','lifeboat_station'); +-- SELECT assert_tag_not_empty('water', 'emergency','marine_rescue'); +-- SELECT assert_tag_not_empty('water', 'emergency','water_rescue'); +-- SELECT assert_tag_not_empty('water', 'man_made','offshore_platform'); +-- SELECT assert_tag_not_empty('water', 'man_made','spring'); +-- SELECT assert_tag_not_empty('water', 'historic','aquaduct'); + +-- Relating to man_made=water_well +--SELECT assert_map_not_empty('water', 'pump:'); +SELECT assert_col_not_empty('water', 'pump'); +SELECT assert_col_not_empty('water', 'drinking_water'); +-- SELECT assert_col_not_empty('water', 'handle'); +SELECT assert_col_not_empty('water', 'mechanical_driver'); +SELECT assert_col_not_empty('water', 'depth');