diff --git a/.github/actions/ubuntu-prerequisites/action.yml b/.github/actions/ubuntu-prerequisites/action.yml index 12743319a..8a7eb0c91 100644 --- a/.github/actions/ubuntu-prerequisites/action.yml +++ b/.github/actions/ubuntu-prerequisites/action.yml @@ -16,10 +16,12 @@ runs: - name: Install software run: | sudo apt-get install -yq --no-install-suggests --no-install-recommends \ + cimg-dev \ libboost-filesystem-dev \ libboost-system-dev \ libbz2-dev \ libexpat1-dev \ + libpotrace-dev \ libpq-dev \ libproj-dev \ pandoc \ diff --git a/.github/actions/win-install/action.yml b/.github/actions/win-install/action.yml index bf5171099..3ebc475b3 100644 --- a/.github/actions/win-install/action.yml +++ b/.github/actions/win-install/action.yml @@ -5,9 +5,8 @@ runs: steps: - name: Install packages - run: vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows proj4:x64-windows boost-geometry:x64-windows boost-system:x64-windows boost-filesystem:x64-windows boost-property-tree:x64-windows lua:x64-windows libpq:x64-windows + run: vcpkg install cimg:x64-windows bzip2:x64-windows expat:x64-windows zlib:x64-windows proj4:x64-windows boost-geometry:x64-windows boost-system:x64-windows boost-filesystem:x64-windows boost-property-tree:x64-windows lua:x64-windows libpq:x64-windows shell: bash - - name: Install psycopg2 and beahve run: python -m pip install psycopg2 behave shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b76af0dc6..45a1bcde7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: - name: Install prerequisites run: | - brew install lua boost postgis pandoc + brew install lua boost postgis pandoc cimg potrace pip3 install psycopg2 behave pg_ctl -D /usr/local/var/postgres init pg_ctl -D /usr/local/var/postgres start @@ -45,6 +45,7 @@ jobs: env: CC: gcc-10 CXX: g++-10 + EXTRA_FLAGS: -Wno-unused-but-set-parameter # workaround for GCC bug LUA_VERSION: 5.3 LUAJIT_OPTION: ON POSTGRESQL_VERSION: 9.6 @@ -79,6 +80,7 @@ jobs: env: CC: gcc-10 CXX: g++-10 + EXTRA_FLAGS: -Wno-unused-but-set-parameter # workaround for GCC bug LUA_VERSION: 5.3 LUAJIT_OPTION: OFF POSTGRESQL_VERSION: 10 diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index 3fe0d6790..cba48c0f9 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -37,12 +37,14 @@ jobs: run: | sudo apt-get purge -yq postgresql* sudo apt-get install -yq --no-install-suggests --no-install-recommends \ + cimg-dev \ libboost-filesystem-dev \ libboost-system-dev \ libbz2-dev \ libexpat1-dev \ liblua${LUA_VERSION}-dev \ libluajit-5.1-dev \ + libpotrace-dev \ libpq-dev \ libproj-dev \ lua${LUA_VERSION} \ diff --git a/CMakeLists.txt b/CMakeLists.txt index f92a35771..2d7552c08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,6 +205,11 @@ include_directories(SYSTEM ${PostgreSQL_INCLUDE_DIRS}) find_package(Threads) +find_path(POTRACE_INCLUDE_DIR potracelib.h) +find_library(POTRACE_LIBRARY NAMES potrace) + +find_path(CIMG_INCLUDE_DIR CImg.h) + ############### Libraries are found now ######################## set(LIBS ${Boost_LIBRARIES} ${PostgreSQL_LIBRARY} ${OSMIUM_LIBRARIES}) @@ -276,6 +281,32 @@ add_subdirectory(src) add_executable(osm2pgsql src/osm2pgsql.cpp) target_link_libraries(osm2pgsql osm2pgsql_lib ${LIBS}) +if (${POTRACE_LIBRARY} STREQUAL "POTRACE_LIBRARY-NOTFOUND" OR ${CIMG_INCLUDE_DIR} STREQUAL "CIMG_INCLUDE_DIR-NOTFOUND") + message(STATUS "Did not find cimg and/or potrace library. Not building osm2pgsql-gen.") +else() + if (WITH_LUA) + message(STATUS "Found cimg and potrace library. Building osm2pgsql-gen.") + include_directories(SYSTEM ${CIMG_INCLUDE_DIR}) + include_directories(SYSTEM ${POTRACE_INCLUDE_DIR}) + add_executable(osm2pgsql-gen src/gen/osm2pgsql-gen.cpp + src/gen/canvas.cpp + src/gen/gen-base.cpp + src/gen/gen-create.cpp + src/gen/gen-discrete-isolation.cpp + src/gen/gen-rivers.cpp + src/gen/gen-tile-builtup.cpp + src/gen/gen-tile-raster.cpp + src/gen/gen-tile-vector.cpp + src/gen/gen-tile.cpp + src/gen/params.cpp + src/gen/raster.cpp + src/gen/tracer.cpp) + target_link_libraries(osm2pgsql-gen osm2pgsql_lib ${LIBS} ${POTRACE_LIBRARY}) + else() + message(STATUS "No Lua. Not building osm2pgsql-gen.") + endif() +endif() + ############################################################# # Optional "clang-tidy" target ############################################################# @@ -287,7 +318,7 @@ find_program(CLANG_TIDY if (CLANG_TIDY) message(STATUS "Looking for clang-tidy - found ${CLANG_TIDY}") - file(GLOB CT_CHECK_FILES src/*.cpp tests/*cpp) + file(GLOB CT_CHECK_FILES src/*.cpp src/*/*.cpp tests/*cpp) add_custom_target(clang-tidy ${CLANG_TIDY} diff --git a/README.md b/README.md index 3264c0ad3..db1b67287 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ Required libraries are * [zlib](https://www.zlib.net/) * [Boost libraries](https://www.boost.org/), including geometry, system and filesystem +* [CImg](https://cimg.eu/) (Optional, see README-gen.md) +* [potrace](https://potrace.sourceforge.net/) (Optional, see README-gen.md) * [PostgreSQL](https://www.postgresql.org/) client libraries * [Lua](https://www.lua.org/) (Optional, used for Lua tag transforms and the flex output) @@ -80,7 +82,7 @@ On a Debian or Ubuntu system, this can be done with: ```sh sudo apt-get install make cmake g++ libboost-dev libboost-system-dev \ - libboost-filesystem-dev libexpat1-dev zlib1g-dev \ + libboost-filesystem-dev libexpat1-dev zlib1g-dev libpotrace-dev cimg-dev \ libbz2-dev libpq-dev libproj-dev lua5.3 liblua5.3-dev pandoc ``` @@ -88,6 +90,7 @@ On a Fedora system, use ```sh sudo dnf install cmake make gcc-c++ boost-devel expat-devel zlib-devel \ + potrace-devel cimg-devel \ bzip2-devel postgresql-devel proj-devel proj-epsg lua-devel pandoc ``` diff --git a/src/gen/canvas.cpp b/src/gen/canvas.cpp new file mode 100644 index 000000000..faa402e54 --- /dev/null +++ b/src/gen/canvas.cpp @@ -0,0 +1,130 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "canvas.hpp" +#include "raster.hpp" + +cimg_library::CImg canvas_t::create_pointlist(geom::point_list_t const &pl, + tile_t const &tile) const +{ + cimg_library::CImg points{static_cast(pl.size()), 2}; + + int n = 0; + for (auto const point : pl) { + auto const tp = tile.to_tile_coords(point, m_extent); + points(n, 0) = static_cast(static_cast(m_buffer) + tp.x()); + points(n, 1) = + static_cast(static_cast(m_buffer + m_extent) - tp.y()); + ++n; + } + + return points; +} + +std::size_t canvas_t::draw_polygon(geom::polygon_t const &polygon, + tile_t const &tile) +{ + if (polygon.inners().empty()) { + m_rast.draw_polygon(create_pointlist(polygon.outer(), tile), &White); + return polygon.outer().size(); + } + + std::size_t num_points = polygon.outer().size(); + m_temp.draw_polygon(create_pointlist(polygon.outer(), tile), &White); + for (auto const &inner : polygon.inners()) { + num_points += inner.size(); + m_temp.draw_polygon(create_pointlist(inner, tile), &Black); + } + m_rast |= m_temp; + + return num_points; +} + +std::size_t canvas_t::draw_linestring(geom::linestring_t const &linestring, + tile_t const &tile) +{ + m_rast.draw_line(create_pointlist(linestring, tile), &White); + return linestring.size(); +} + +std::size_t canvas_t::draw(geom::geometry_t const &geometry, tile_t const &tile) +{ + if (geometry.is_linestring()) { + auto const &linestring = geometry.get(); + return draw_linestring(linestring, tile); + } + + if (geometry.is_polygon()) { + auto const &polygon = geometry.get(); + return draw_polygon(polygon, tile); + } + + if (geometry.is_multipolygon()) { + auto const &mp = geometry.get(); + std::size_t num_points = 0; + for (auto const &p : mp) { + num_points += draw_polygon(p, tile); + } + return num_points; + } + + // XXX other geometry types? + + return 0; +} + +void canvas_t::save(std::string const &filename) const +{ + m_rast.save(filename.c_str()); +} + +std::string canvas_t::to_wkb(tile_t const &tile, double margin) const +{ + std::string wkb; + wkb.reserve(61 + 2 + m_rast.size()); + + // header + wkb_raster_header header{}; + header.nBands = 1; + header.scaleX = tile.extent() / static_cast(m_extent); + header.scaleY = -header.scaleX; + header.ipX = tile.xmin(margin); + header.ipY = tile.ymax(margin); + header.width = m_extent + 2 * m_buffer; + header.height = header.width; + add_raster_header(&wkb, header); + + // band + wkb_raster_band band{}; + band.bits = 4; + add_raster_band(&wkb, band); + + // rasterdata + wkb.append(reinterpret_cast(m_rast.data()), m_rast.size()); + + assert(wkb.size() == 61 + 2 + m_rast.size()); + + return wkb; +} + +void canvas_t::merge(canvas_t const &other) { m_rast |= other.m_rast; } + +std::string to_hex(std::string const &in) +{ + std::string result; + char const *const lookup_hex = "0123456789ABCDEF"; + + for (const auto c : in) { + unsigned int const num = static_cast(c); + result += lookup_hex[(num >> 4U) & 0xfU]; + result += lookup_hex[num & 0xfU]; + } + + return result; +} diff --git a/src/gen/canvas.hpp b/src/gen/canvas.hpp new file mode 100644 index 000000000..bc439b947 --- /dev/null +++ b/src/gen/canvas.hpp @@ -0,0 +1,88 @@ +#ifndef OSM2PGSQL_CANVAS_HPP +#define OSM2PGSQL_CANVAS_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "geom.hpp" +#include "tile.hpp" + +#define cimg_display 0 // NOLINT(cppcoreguidelines-macro-usage) +#include "CImg.h" + +#include + +/** + * This class wraps the image class from the CImg library. + */ +class canvas_t +{ +public: + static void info() { cimg_library::cimg::info(); } + + /** + * Create a new image canvas. It will be quadratic and have the width and + * height extent + 2*buffer. + */ + canvas_t(std::size_t extent, std::size_t buffer) + : m_extent(extent), + m_buffer(buffer), m_rast{size(), size(), 1, 1, 0}, m_temp{size(), size(), + 1, 1, 0} + {} + + unsigned int size() const noexcept + { + return static_cast(m_extent + 2 * m_buffer); + } + + unsigned char const *begin() const noexcept { return m_rast.begin(); } + unsigned char const *end() const noexcept { return m_rast.end(); } + + std::size_t draw(geom::geometry_t const &geometry, tile_t const &tile); + + unsigned char operator()(int x, int y) const noexcept + { + return m_rast(x, y, 0, 0); + } + + void open_close(unsigned int buffer_size) + { + m_rast.dilate(buffer_size).erode(buffer_size * 2).dilate(buffer_size); + } + + void save(std::string const &filename) const; + + std::string to_wkb(tile_t const &tile, double margin) const; + + void merge(canvas_t const &other); + +private: + constexpr static unsigned char const Black = 0; + constexpr static unsigned char const White = 255; + + using image_type = cimg_library::CImg; + + cimg_library::CImg create_pointlist(geom::point_list_t const &pl, + tile_t const &tile) const; + + std::size_t draw_polygon(geom::polygon_t const &polygon, + tile_t const &tile); + + std::size_t draw_linestring(geom::linestring_t const &linestring, + tile_t const &tile); + + std::size_t m_extent; + std::size_t m_buffer; + image_type m_rast; + image_type m_temp; +}; // class canvas_t + +std::string to_hex(std::string const &in); + +#endif // OSM2PGSQL_CANVAS_HPP diff --git a/src/gen/gen-base.cpp b/src/gen/gen-base.cpp new file mode 100644 index 000000000..0aa728371 --- /dev/null +++ b/src/gen/gen-base.cpp @@ -0,0 +1,108 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-base.hpp" + +#include "format.hpp" +#include "params.hpp" + +#include + +gen_base_t::gen_base_t(pg_conn_t *connection, params_t *params) +: m_connection(connection), m_params(params) +{ + assert(connection); + assert(params); + + auto const schema = params->get_identifier("schema"); + if (schema.empty()) { + params->set("schema", "public"); + } + + if (params->has("src_table")) { + auto const src_table = get_params().get_identifier("src_table"); + params->set("src", qualified_name(schema, src_table)); + } + + if (params->has("dest_table")) { + auto const dest_table = get_params().get_identifier("dest_table"); + params->set("dest", qualified_name(schema, dest_table)); + } + + if (!params->has("geom_column")) { + params->set("geom_column", "geom"); + } + + m_debug = get_params().get_bool("debug", false); +} + +std::string gen_base_t::name() { return get_params().get_string("name", ""); } + +static pg_result_t dbexec_internal( + pg_conn_t const &connection, std::string const &templ, + fmt::dynamic_format_arg_store const &format_store) +{ + try { + auto const sql = fmt::vformat(templ, format_store); + return connection.exec(sql); + } catch (fmt::format_error const &e) { + log_error("Missing parameter for template: '{}'", templ); + throw; + } +} + +pg_result_t gen_base_t::dbexec(std::string const &templ) +{ + fmt::dynamic_format_arg_store format_store; + for (auto const &[key, value] : get_params()) { + format_store.push_back(fmt::arg(key.c_str(), to_string(value))); + } + return dbexec_internal(connection(), templ, format_store); +} + +pg_result_t gen_base_t::dbexec(params_t const &tmp_params, + std::string const &templ) +{ + fmt::dynamic_format_arg_store format_store; + for (auto const &[key, value] : get_params()) { + format_store.push_back(fmt::arg(key.c_str(), to_string(value))); + } + for (auto const &[key, value] : tmp_params) { + format_store.push_back(fmt::arg(key.c_str(), to_string(value))); + } + return dbexec_internal(connection(), templ, format_store); +} + +void gen_base_t::raster_table_preprocess(std::string const &table) +{ + params_t tmp_params; + tmp_params.set("TABLE", table); + + dbexec(tmp_params, "SELECT DropRasterConstraints('{schema}'::name," + " '{TABLE}'::name, 'rast'::name)"); +} + +void gen_base_t::raster_table_postprocess(std::string const &table) +{ + params_t tmp_params; + tmp_params.set("TABLE", table); + + dbexec(tmp_params, R"(SELECT AddRasterConstraints('{schema}'::name,)" + R"( '{TABLE}'::name, 'rast'::name))"); + dbexec(tmp_params, R"(ALTER TABLE "{schema}"."{TABLE}")" + R"( VALIDATE CONSTRAINT enforce_max_extent_rast)"); + dbexec(tmp_params, R"(ANALYZE "{schema}"."{TABLE}")"); +} + +void gen_base_t::merge_timers(gen_base_t const &other) +{ + for (std::size_t n = 0; n < m_timers.size(); ++n) { + m_timers[n] += other.m_timers[n]; + } +} diff --git a/src/gen/gen-base.hpp b/src/gen/gen-base.hpp new file mode 100644 index 000000000..b6c6f32d9 --- /dev/null +++ b/src/gen/gen-base.hpp @@ -0,0 +1,95 @@ +#ifndef OSM2PGSQL_GEN_BASE_HPP +#define OSM2PGSQL_GEN_BASE_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "logging.hpp" +#include "pgsql.hpp" +#include "util.hpp" + +#include +#include +#include + +class params_t; +class tile_t; + +/** + * Base class for generalization strategies. + */ +class gen_base_t +{ +public: + virtual ~gen_base_t() = default; + + /// Process data. Used for non-tile-based generalizers. + virtual void process() {} + + /// Process one tile. Used for tile-based generalizers. + virtual void process(tile_t const & /*tile*/) {} + + /// Optional postprocessing after all tiles. + virtual void post() {} + + /// Get the name of the generalization strategy. + virtual std::string_view strategy() const noexcept = 0; + + virtual bool on_tiles() const noexcept { return false; } + + void merge_timers(gen_base_t const &other); + + std::vector const &timers() const noexcept + { + return m_timers; + } + + bool debug() const noexcept { return m_debug; } + + std::string name(); + + template + void log_gen(ARGS... args) + { + if (m_debug) { + log_debug(args...); + } + } + +protected: + gen_base_t(pg_conn_t *connection, params_t *params); + + pg_conn_t &connection() noexcept { return *m_connection; } + + std::size_t add_timer(char const *name) + { + m_timers.emplace_back(name); + return m_timers.size() - 1; + } + + util::timer_t &timer(std::size_t n) noexcept { return m_timers[n]; } + + params_t const &get_params() const noexcept { return *m_params; } + + pg_result_t dbexec(std::string const &templ); + + pg_result_t dbexec(params_t const &tmp_params, std::string const &templ); + + void raster_table_preprocess(std::string const &table); + + void raster_table_postprocess(std::string const &table); + +private: + std::vector m_timers; + pg_conn_t *m_connection; + params_t *m_params; + bool m_debug = false; +}; // class gen_base_t + +#endif // OSM2PGSQL_GEN_BASE_HPP diff --git a/src/gen/gen-create.cpp b/src/gen/gen-create.cpp new file mode 100644 index 000000000..4486338a0 --- /dev/null +++ b/src/gen/gen-create.cpp @@ -0,0 +1,45 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-create.hpp" + +#include "gen-discrete-isolation.hpp" +#include "gen-rivers.hpp" +#include "gen-tile-builtup.hpp" +#include "gen-tile-raster.hpp" +#include "gen-tile-vector.hpp" +#include "params.hpp" + +std::unique_ptr create_generalizer(std::string const &strategy, + pg_conn_t *connection, + params_t *params) +{ + auto generalizer = [&]() -> std::unique_ptr { + if (strategy == "builtup") { + return std::make_unique(connection, params); + } + if (strategy == "discrete-isolation") { + return std::make_unique(connection, params); + } + if (strategy == "raster-union") { + return std::make_unique(connection, + params); + } + if (strategy == "rivers") { + return std::make_unique(connection, params); + } + if (strategy == "vector-union") { + return std::make_unique(connection, + params); + } + throw fmt_error("Unknown generalization strategy '{}'.", strategy); + }(); + + return generalizer; +} diff --git a/src/gen/gen-create.hpp b/src/gen/gen-create.hpp new file mode 100644 index 000000000..54dacbef4 --- /dev/null +++ b/src/gen/gen-create.hpp @@ -0,0 +1,26 @@ +#ifndef OSM2PGSQL_GEN_CREATE_HPP +#define OSM2PGSQL_GEN_CREATE_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-base.hpp" + +#include +#include + +class params_t; +class pg_conn_t; + +/// Instantiate a generalizer for the specified strategy. +std::unique_ptr create_generalizer(std::string const &strategy, + pg_conn_t *connection, + params_t *params); + +#endif // OSM2PGSQL_GEN_CREATE_HPP diff --git a/src/gen/gen-discrete-isolation.cpp b/src/gen/gen-discrete-isolation.cpp new file mode 100644 index 000000000..7f50c5be5 --- /dev/null +++ b/src/gen/gen-discrete-isolation.cpp @@ -0,0 +1,147 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-discrete-isolation.hpp" + +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "util.hpp" + +#include +#include + +gen_di_t::gen_di_t(pg_conn_t *connection, params_t *params) +: gen_base_t(connection, params), m_timer_get(add_timer("get")), + m_timer_sort(add_timer("sort")), m_timer_di(add_timer("di")), + m_timer_reorder(add_timer("reorder")), m_timer_write(add_timer("write")) +{ + params->check_identifier_with_default("id_column", "id"); + params->check_identifier_with_default("importance_column", "importance"); +} + +void gen_di_t::process() +{ + struct feature + { + // input: unique id of the feature + uint64_t id; + + // input: importance of the feature (positive, larger is more imporant) + double importance; + + // input: x/y coordinate of the feature + double x; + double y; + + // output: discrete isolation + double di; + + // output: rank for importance + uint32_t irank; + }; + + log_gen("Reading data from database..."); + + std::vector data; + timer(m_timer_get).start(); + { + auto const result = dbexec(R"( +SELECT {id_column}, {importance_column}, + ST_X({geom_column}), ST_Y({geom_column}) +FROM {src} WHERE {importance_column} > 0 +)"); + + data.reserve(result.num_tuples()); + for (int i = 0; i < result.num_tuples(); ++i) { + data.push_back({std::strtoull(result.get_value(i, 0), nullptr, 10), + std::strtod(result.get_value(i, 1), nullptr), + std::strtod(result.get_value(i, 2), nullptr), + std::strtod(result.get_value(i, 3), nullptr), 0.0, + 0}); + } + } + timer(m_timer_get).stop(); + log_gen("Read {} features", data.size()); + + if (data.size() < 2) { + log_gen("Found fewer than two features. Nothing to do."); + return; + } + + log_gen("Sorting data by importance..."); + timer(m_timer_sort).start(); + { + std::sort(data.begin(), data.end(), + [](feature const &a, feature const &b) noexcept { + return a.importance > b.importance; + }); + { + uint32_t n = 0; + for (auto &item : data) { + item.irank = n++; + } + } + } + timer(m_timer_sort).stop(); + + log_gen("Calculating discrete isolation..."); + timer(m_timer_di).start(); + { + std::vector> coords; + coords.reserve(data.size()); + for (auto const &d : data) { + coords.emplace_back(d.x, d.y); + } + + for (std::size_t n = 1; n < data.size(); ++n) { + if (n % 10000 == 0) { + log_gen(" {}", n); + } + double min = 100000000000000.0; + for (std::size_t m = 0; m < n; ++m) { + double const dx = coords[m].first - coords[n].first; + double const dy = coords[m].second - coords[n].second; + double const dist = dx * dx + dy * dy; + if (dist < min) { + min = dist; + } + } + data[n].di = sqrt(min); + } + data[0].di = data[1].di + 1; + } + timer(m_timer_di).stop(); + + log_gen("Sorting data by discrete isolation..."); + timer(m_timer_reorder).start(); + std::sort(data.begin(), data.end(), + [](feature const &a, feature const &b) noexcept { + return a.di > b.di; + }); + timer(m_timer_reorder).stop(); + + log_gen("Writing results to destination table..."); + dbexec("PREPARE update (int, real, int4, int8) AS" + " UPDATE {src} SET dirank = $1, discr_iso = $2, irank = $3" + " WHERE {id_column} = $4"); + + timer(m_timer_write).start(); + connection().exec("BEGIN"); + std::size_t n = 0; + for (auto const &d : data) { + connection().exec_prepared("update", n++, d.di, d.irank, d.id); + } + connection().exec("COMMIT"); + timer(m_timer_write).stop(); + + dbexec("ANALYZE {src}"); + + log_gen("Done."); +} diff --git a/src/gen/gen-discrete-isolation.hpp b/src/gen/gen-discrete-isolation.hpp new file mode 100644 index 000000000..90b5ba578 --- /dev/null +++ b/src/gen/gen-discrete-isolation.hpp @@ -0,0 +1,37 @@ +#ifndef OSM2PGSQL_GEN_DISCRETE_ISOLATION_HPP +#define OSM2PGSQL_GEN_DISCRETE_ISOLATION_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-base.hpp" + +#include + +class gen_di_t : public gen_base_t +{ +public: + gen_di_t(pg_conn_t *connection, params_t *params); + + void process() override; + + std::string_view strategy() const noexcept override + { + return "discrete-isolation"; + } + +private: + std::size_t m_timer_get; + std::size_t m_timer_sort; + std::size_t m_timer_di; + std::size_t m_timer_reorder; + std::size_t m_timer_write; +}; + +#endif // OSM2PGSQL_GEN_DISCRETE_ISOLATION_HPP diff --git a/src/gen/gen-rivers.cpp b/src/gen/gen-rivers.cpp new file mode 100644 index 000000000..0835bdd16 --- /dev/null +++ b/src/gen/gen-rivers.cpp @@ -0,0 +1,347 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-rivers.hpp" + +#include "geom-functions.hpp" +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "util.hpp" +#include "wkb.hpp" + +#include +#include +#include +#include +#include + +gen_rivers_t::gen_rivers_t(pg_conn_t *connection, params_t *params) +: gen_base_t(connection, params), m_timer_area(add_timer("area")), + m_timer_prep(add_timer("prep")), m_timer_get(add_timer("get")), + m_timer_sort(add_timer("sort")), m_timer_net(add_timer("net")), + m_timer_remove(add_timer("remove")), m_timer_width(add_timer("width")), + m_timer_write(add_timer("write")), + m_delete_existing(params->has("delete_existing")) +{ + params->check_identifier_with_default("src_areas", "waterway_areas"); + params->check_identifier_with_default("id_column", "way_id"); + params->check_identifier_with_default("width_column", "width"); + params->check_identifier_with_default("name_column", "name"); + + params->set("qualified_src_areas", + qualified_name(get_params().get_string("schema"), + get_params().get_string("src_areas"))); +} + +/// The data for a graph edge in the waterway network. +struct edge_t +{ + // All the points in this edge + geom::linestring_t points; + + // Edges can be made from (part) of one or more OSM ways, this is the id + // of one of them. + osmid_t id = 0; + + // The width of the river along this edge + double width = 0.0; +}; + +bool operator<(edge_t const &a, edge_t const &b) noexcept +{ + assert(a.points.size() > 1 && b.points.size() > 1); + if (a.points[0] == b.points[0]) { + return a.points[1] < b.points[1]; + } + return a.points[0] < b.points[0]; +} + +bool operator<(edge_t const &a, geom::point_t b) noexcept +{ + assert(!a.points.empty()); + return a.points[0] < b; +} + +bool operator<(geom::point_t a, edge_t const &b) noexcept +{ + assert(!b.points.empty()); + return a < b.points[0]; +} + +static void +follow_chain_and_set_width(edge_t const &edge, std::vector *edges, + std::map const &node_order, + geom::linestring_t *seen) +{ + assert(!edge.points.empty()); + + auto const seen_it = + std::find(seen->cbegin(), seen->cend(), edge.points[0]); + if (seen_it != seen->cend()) { + return; // loop detected + } + + seen->push_back(edge.points[0]); + + assert(edge.points.size() > 1); + auto const next_point = edge.points.back(); + if (node_order.at(next_point) > 1) { + auto const [s, e] = + std::equal_range(edges->begin(), edges->end(), next_point); + + if (std::next(s) == e) { + if (s->width < edge.width) { + s->width = edge.width; + follow_chain_and_set_width(*s, edges, node_order, seen); + } + } else { + for (auto it = s; it != e; ++it) { + assert(it->points[0] == next_point); + if (it->width < edge.width) { + it->width = edge.width; + auto seen2 = *seen; + follow_chain_and_set_width(*it, edges, node_order, &seen2); + } + } + } + } +} + +static void assemble_edge(edge_t *edge, std::vector *edges, + std::map const &node_order) + +{ + assert(edge); + assert(edges); + while (true) { + assert(edge->points.size() > 1); + geom::point_t const next_point = edge->points.back(); + + auto const count = node_order.at(next_point); + if (count != 2) { + return; + } + + auto const [s, e] = + std::equal_range(edges->begin(), edges->end(), next_point); + + if (s == e) { + return; + } + assert(e == std::next(s)); + + auto const it = s; + if (it->points.size() == 1 || &*it == edge) { + return; + } + + if (it->points[0] != next_point) { + return; + } + assert(it != edges->end()); + + edge->width = std::max(edge->width, it->width); + + if (it->points.size() == 2) { + edge->points.push_back(it->points.back()); + it->points.resize(1); + it->points.shrink_to_fit(); + } else { + edge->points.insert(edge->points.end(), + std::next(it->points.begin()), + it->points.end()); + it->points.resize(1); + it->points.shrink_to_fit(); + return; + } + } +} + +/// Get some stats from source table +void gen_rivers_t::get_stats() +{ + auto const result = + dbexec("SELECT count(*), sum(ST_NumPoints(geom)) FROM {src}"); + + m_num_waterways = strtoul(result.get_value(0, 0), nullptr, 10); + m_num_points = strtoul(result.get_value(0, 1), nullptr, 10); + + log_gen("Found {} waterways with {} points.", m_num_waterways, + m_num_points); +} + +static std::string const & +get_name(std::unordered_map const &names, osmid_t id) +{ + static std::string const empty; + auto const it = names.find(id); + if (it == names.end()) { + return empty; + } + return it->second; +} + +void gen_rivers_t::process() +{ + log_gen("Calculate waterway area width..."); + timer(m_timer_area).start(); + dbexec(R"(UPDATE {qualified_src_areas} SET width =)" + R"( (ST_MaximumInscribedCircle("{geom_column}")).radius * 2)" + R"( WHERE width IS NULL)"); + dbexec("ANALYZE {qualified_src_areas}"); + timer(m_timer_area).stop(); + + log_gen("Get 'width' from areas onto lines..."); + timer(m_timer_prep).start(); + dbexec(R"( +WITH _covered_lines AS ( + SELECT "{geom_column}" AS geom, "{id_column}" AS wid FROM {src} w + WHERE ST_NumPoints(w."{geom_column}") > 2 AND ST_CoveredBy(w."{geom_column}", + (SELECT ST_Union("{geom_column}") FROM {qualified_src_areas} a + WHERE ST_Intersects(w."{geom_column}", a."{geom_column}"))) +), _intersections AS ( + SELECT w.wid, ST_Intersection(a.geom, w.geom) AS inters, + ST_Length(w.geom) AS wlength, a.width AS width + FROM _covered_lines w, {qualified_src_areas} a + WHERE ST_Intersects(w.geom, a.geom) +), _lines AS ( + SELECT wid, wlength, ST_Length(inters) * width AS lenwidth FROM _intersections + WHERE ST_GeometryType(inters) IN ('ST_LineString', 'ST_MultiLineString') +), _glines AS ( + SELECT wid, sum(lenwidth) / wlength AS width FROM _lines + GROUP BY wid, wlength +) +UPDATE {src} a SET width = l.width + FROM _glines l WHERE l.wid = a."{id_column}" AND a.width IS NULL + )"); + timer(m_timer_prep).stop(); + + log_gen("Reading waterway lines from database..."); + get_stats(); + + // This vector will initially contain all segments (connection between + // two points) from waterway ways. They will later be assembled into + // graph edges connecting points where the waterways network branches. + std::vector edges; + edges.reserve(m_num_points - m_num_waterways); + + // This stores the order of each node in our graph, i.e. the number of + // connections this node has. Order 1 are beginning or end of a waterway, + // order 2 is just the continuing waterway, order >= 3 is a branching + // point. + std::map node_order; + + // This is where we keep the names of all waterways indexed by their + // way id. + std::unordered_map names; + + timer(m_timer_get).start(); + { + auto const result = dbexec(R"( +SELECT "{id_column}", "{width_column}", "{name_column}", "{geom_column}" + FROM {src}; +)"); + + for (int i = 0; i < result.num_tuples(); ++i) { + auto const id = std::strtol(result.get_value(i, 0), nullptr, 10); + auto const width = std::strtod(result.get_value(i, 1), nullptr); + auto const name = result.get(i, 2); + if (!name.empty()) { + names.emplace(id, name); + } + auto const geom = ewkb_to_geom(decode_hex(result.get_value(i, 3))); + + if (geom.is_linestring()) { + auto const &ls = geom.get(); + geom::for_each_segment(ls, + [&](geom::point_t a, geom::point_t b) { + if (a != b) { + auto &f = edges.emplace_back(); + f.points.push_back(a); + f.points.push_back(b); + f.id = id; + f.width = width; + node_order[a]++; + node_order[b]++; + } + }); + } + } + } + timer(m_timer_get).stop(); + log_gen("Read {} segments, {} unique points, and {} names.", edges.size(), + node_order.size(), names.size()); + + if (edges.size() < 2) { + log_gen("Found fewer than two segments. Nothing to do."); + return; + } + + log_gen("Sorting segments..."); + timer(m_timer_sort).start(); + std::sort(edges.begin(), edges.end()); + timer(m_timer_sort).stop(); + + log_gen("Assembling edges from segments..."); + timer(m_timer_net).start(); + for (auto &edge : edges) { + if (edge.points.size() > 1) { + assemble_edge(&edge, &edges, node_order); + } + } + timer(m_timer_net).stop(); + + log_gen("Removing now empty edges..."); + timer(m_timer_remove).start(); + { + auto const last = + std::remove_if(edges.begin(), edges.end(), [](edge_t const &edge) { + return edge.points.size() == 1; + }); + edges.erase(last, edges.end()); + std::sort(edges.begin(), edges.end()); + } + timer(m_timer_remove).stop(); + + log_gen("Network has {} edges.", edges.size()); + + log_gen("Propagating 'width' property downstream..."); + timer(m_timer_width).start(); + for (auto &edge : edges) { + assert(!edge.points.empty()); + geom::linestring_t seen; + follow_chain_and_set_width(edge, &edges, node_order, &seen); + } + timer(m_timer_width).stop(); + + if (m_delete_existing) { + dbexec("TRUNCATE {dest}"); + } + + log_gen("Writing results to destination table..."); + dbexec("PREPARE ins (int8, real, text, geometry) AS" + " INSERT INTO {dest} ({id_column}, width, name, geom)" + " VALUES ($1, $2, $3, $4)"); + + timer(m_timer_write).start(); + connection().exec("BEGIN"); + for (auto &edge : edges) { + geom::geometry_t const geom{std::move(edge.points), 3857}; + auto const wkb = geom_to_ewkb(geom); + connection().exec_prepared("ins", edge.id, edge.width, + get_name(names, edge.id), binary_param(wkb)); + } + connection().exec("COMMIT"); + timer(m_timer_write).stop(); + + dbexec("ANALYZE {dest}"); + + log_gen("Done."); +} diff --git a/src/gen/gen-rivers.hpp b/src/gen/gen-rivers.hpp new file mode 100644 index 000000000..f59afe72a --- /dev/null +++ b/src/gen/gen-rivers.hpp @@ -0,0 +1,43 @@ +#ifndef OSM2PGSQL_GEN_RIVERS_HPP +#define OSM2PGSQL_GEN_RIVERS_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-base.hpp" + +#include + +class gen_rivers_t : public gen_base_t +{ +public: + gen_rivers_t(pg_conn_t *connection, params_t *params); + + void process() override; + + std::string_view strategy() const noexcept override { return "rivers"; } + +private: + void get_stats(); + + std::size_t m_timer_area; + std::size_t m_timer_prep; + std::size_t m_timer_get; + std::size_t m_timer_sort; + std::size_t m_timer_net; + std::size_t m_timer_remove; + std::size_t m_timer_width; + std::size_t m_timer_write; + + std::size_t m_num_waterways = 0; + std::size_t m_num_points = 0; + bool m_delete_existing; +}; + +#endif // OSM2PGSQL_GEN_RIVERS_HPP diff --git a/src/gen/gen-tile-builtup.cpp b/src/gen/gen-tile-builtup.cpp new file mode 100644 index 000000000..96fb22e96 --- /dev/null +++ b/src/gen/gen-tile-builtup.cpp @@ -0,0 +1,280 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile-builtup.hpp" + +#include "canvas.hpp" +#include "geom-functions.hpp" +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "raster.hpp" +#include "tile.hpp" +#include "tracer.hpp" +#include "wkb.hpp" + +#include + +static std::size_t round_up(std::size_t value, std::size_t multiple) noexcept +{ + return ((value + multiple - 1U) / multiple) * multiple; +} + +gen_tile_builtup_t::gen_tile_builtup_t(pg_conn_t *connection, params_t *params) +: gen_tile_t(connection, params), m_timer_draw(add_timer("draw")), + m_timer_simplify(add_timer("simplify")), + m_timer_vectorize(add_timer("vectorize")), m_timer_write(add_timer("write")) +{ + m_schema = get_params().get_identifier("schema"); + m_source_tables = + osmium::split_string(get_params().get_string("src_tables"), ','); + + m_margin = get_params().get_double("margin"); + m_image_extent = uint_in_range(*params, "image_extent", 1024, 65536, 2048); + m_image_buffer = + uint_in_range(*params, "image_buffer", 0, m_image_extent, 0); + + auto const buffer_sizes = + osmium::split_string(get_params().get_string("buffer_size"), ','); + for (auto const &bs : buffer_sizes) { + m_buffer_sizes.push_back(std::strtoul(bs.c_str(), nullptr, 10)); + } + + m_turdsize = static_cast( + uint_in_range(*params, "turdsize", 0, 65536, m_turdsize)); + m_min_area = get_params().get_double("min_area", 0.0); + + if (get_params().has("area_column")) { + m_has_area_column = true; + get_params().get_identifier("area_column"); + } + + if (get_params().has("img_path")) { + m_image_path = get_params().get_string("img_path"); + } + + if (get_params().has("img_table")) { + m_image_table = get_params().get_string("img_table"); + + for (auto const &table : m_source_tables) { + for (char const variant : {'i', 'o'}) { + auto const table_name = + fmt::format("{}_{}_{}", m_image_table, table, variant); + connection->exec(R"( +CREATE TABLE IF NOT EXISTS "{}" ( + id SERIAL PRIMARY KEY NOT NULL, + zoom INT4, + x INT4, + y INT4, + rast RASTER +) +)", + table_name); + raster_table_preprocess(table_name); + } + } + } + + if (params->get_bool("make_valid")) { + params->set( + "geom_sql", + "(ST_Dump(ST_CollectionExtract(ST_MakeValid($1), 3))).geom"); + } else { + params->set("geom_sql", "$1"); + } + + if (m_image_extent < 1024U) { + throw std::runtime_error{"width must be at least 1024"}; + } + + if ((m_image_extent & (m_image_extent - 1)) != 0) { + throw std::runtime_error{"width must be power of 2"}; + } + + m_image_buffer = + round_up(static_cast(m_margin * + static_cast(m_image_extent)), + 64U); + m_margin = static_cast(m_image_buffer) / + static_cast(m_image_extent); + + log_gen("Image extent: {}px, buffer: {}px, margin: {}", m_image_extent, + m_image_buffer, m_margin); + + int n = 0; + for (auto const &src_table : m_source_tables) { + params_t tmp_params; + tmp_params.set("N", std::to_string(n++)); + tmp_params.set("SRC", qualified_name(m_schema, src_table)); + + dbexec(tmp_params, R"( +PREPARE get_geoms_{N} (real, real, real, real) AS + SELECT "{geom_column}", '' AS param + FROM {SRC} + WHERE "{geom_column}" && ST_MakeEnvelope($1, $2, $3, $4, 3857) +)"); + } + + if (m_has_area_column) { + dbexec(R"( +PREPARE insert_geoms (geometry, int, int) AS + INSERT INTO {dest} ("{geom_column}", x, y, "{area_column}") + VALUES ({geom_sql}, $2, $3, $4) +)"); + } else { + dbexec(R"( +PREPARE insert_geoms (geometry, int, int) AS + INSERT INTO {dest} ("{geom_column}", x, y) + VALUES ({geom_sql}, $2, $3) +)"); + } +} + +static void save_image_to_table(pg_conn_t *connection, canvas_t const &canvas, + tile_t const &tile, double margin, + std::string const &table, char const *variant, + std::string const &table_prefix) +{ + auto const wkb = to_hex(canvas.to_wkb(tile, margin)); + + connection->exec("INSERT INTO \"{}_{}_{}\" (zoom, x, y, rast)" + " VALUES ({}, {}, {}, '{}')", + table_prefix, table, variant, tile.zoom(), tile.x(), + tile.y(), wkb); +} + +namespace { + +struct param_canvas_t +{ + canvas_t canvas; + std::string table; +}; + +} // anonymous namespace + +using canvas_list_t = std::vector; + +static void draw_from_db(double margin, canvas_list_t *canvas_list, + pg_conn_t *conn, tile_t const &tile) +{ + int prep = 0; + auto const box = tile.box(margin); + for (auto &cc : *canvas_list) { + std::string const statement = "get_geoms_" + fmt::to_string(prep++); + auto const result = + conn->exec_prepared(statement.c_str(), box.min_x(), box.min_y(), + box.max_x(), box.max_y()); + + for (int n = 0; n < result.num_tuples(); ++n) { + auto const geom = ewkb_to_geom(decode_hex(result.get_value(n, 0))); + cc.canvas.draw(geom, tile); + } + } +} + +void gen_tile_builtup_t::process(tile_t const &tile) +{ + delete_existing(tile); + + canvas_list_t canvas_list; + for (auto const &table : m_source_tables) { + canvas_list.push_back( + {canvas_t{m_image_extent, m_image_buffer}, table}); + } + + if (canvas_list.empty()) { + throw std::runtime_error{"No source tables?!"}; + } + + log_gen("Read from database and draw polygons..."); + timer(m_timer_draw).start(); + draw_from_db(m_margin, &canvas_list, &connection(), tile); + timer(m_timer_draw).stop(); + + std::size_t n = 0; + for (auto &[canvas, table] : canvas_list) { + log_gen("Handling table='{}'", table); + + if (!m_image_path.empty()) { + // Save input images for debugging + save_image_to_file(canvas, tile, m_image_path, table, "i", + m_image_extent, m_margin); + } + + if (!m_image_table.empty()) { + // Store input images in database for debugging + save_image_to_table(&connection(), canvas, tile, m_margin, table, + "i", m_image_table); + } + + if (m_buffer_sizes[n] > 0) { + log_gen("Generalize (buffer={} Mercator units)...", + m_buffer_sizes[n] * tile.extent() / + static_cast(m_image_extent)); + timer(m_timer_simplify).start(); + canvas.open_close(m_buffer_sizes[n]); + timer(m_timer_simplify).stop(); + } + + if (!m_image_path.empty()) { + // Save output image for debugging + save_image_to_file(canvas, tile, m_image_path, table, "o", + m_image_extent, m_margin); + } + + if (!m_image_table.empty()) { + // Store output image in database for debugging + save_image_to_table(&connection(), canvas, tile, m_margin, table, + "o", m_image_table); + } + + ++n; + } + + log_gen("Merge bitmaps..."); + for (std::size_t n = 1; n < canvas_list.size(); ++n) { + canvas_list[0].canvas.merge(canvas_list[n].canvas); + } + + tracer_t tracer{m_image_extent, m_image_buffer, m_turdsize}; + + log_gen("Vectorize..."); + timer(m_timer_vectorize).start(); + auto const geometries = + tracer.trace(canvas_list[0].canvas, tile, m_min_area); + timer(m_timer_vectorize).stop(); + + log_gen("Write geometries to destination table..."); + timer(m_timer_write).start(); + for (auto const &geom : geometries) { + auto const wkb = to_hex(geom_to_ewkb(geom)); + if (m_has_area_column) { + connection().exec_prepared("insert_geoms", wkb, tile.x(), tile.y(), + geom::area(geom)); + } else { + connection().exec_prepared("insert_geoms", wkb, tile.x(), tile.y()); + } + } + timer(m_timer_write).stop(); + log_gen("Inserted {} generalized polygons", geometries.size()); +} + +void gen_tile_builtup_t::post() +{ + if (!m_image_table.empty()) { + for (auto const &table : m_source_tables) { + for (char const variant : {'i', 'o'}) { + raster_table_postprocess( + fmt::format("{}_{}_{}", m_image_table, table, variant)); + } + } + } + dbexec("ANALYZE {dest}"); +} diff --git a/src/gen/gen-tile-builtup.hpp b/src/gen/gen-tile-builtup.hpp new file mode 100644 index 000000000..d130e18f3 --- /dev/null +++ b/src/gen/gen-tile-builtup.hpp @@ -0,0 +1,52 @@ +#ifndef OSM2PGSQL_GEN_TILE_BUILTUP_HPP +#define OSM2PGSQL_GEN_TILE_BUILTUP_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile.hpp" + +#include +#include +#include + +class gen_tile_builtup_t final : public gen_tile_t +{ +public: + gen_tile_builtup_t(pg_conn_t *connection, params_t *params); + + ~gen_tile_builtup_t() override = default; + + void process(tile_t const &tile) override; + + void post() override; + + std::string_view strategy() const noexcept override { return "builtup"; } + +private: + std::size_t m_timer_draw; + std::size_t m_timer_simplify; + std::size_t m_timer_vectorize; + std::size_t m_timer_write; + + std::vector m_source_tables; + std::string m_image_path; + std::string m_schema; + std::string m_dest_table; + std::string m_image_table; + double m_margin = 0.0; + std::size_t m_image_extent = 2048; + std::size_t m_image_buffer = 0; + std::vector m_buffer_sizes; + int m_turdsize = 2; + double m_min_area = 0.0; + bool m_has_area_column; +}; + +#endif // OSM2PGSQL_GEN_TILE_BUILTUP_HPP diff --git a/src/gen/gen-tile-raster.cpp b/src/gen/gen-tile-raster.cpp new file mode 100644 index 000000000..e349f46c2 --- /dev/null +++ b/src/gen/gen-tile-raster.cpp @@ -0,0 +1,249 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile-raster.hpp" + +#include "canvas.hpp" +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "raster.hpp" +#include "tile.hpp" +#include "tracer.hpp" +#include "wkb.hpp" + +#include + +static std::size_t round_up(std::size_t value, std::size_t multiple) noexcept +{ + return ((value + multiple - 1U) / multiple) * multiple; +} + +gen_tile_raster_union_t::gen_tile_raster_union_t(pg_conn_t *connection, + params_t *params) +: gen_tile_t(connection, params), m_timer_draw(add_timer("draw")), + m_timer_simplify(add_timer("simplify")), + m_timer_vectorize(add_timer("vectorize")), m_timer_write(add_timer("write")) +{ + m_margin = get_params().get_double("margin"); + m_image_extent = uint_in_range(*params, "image_extent", 1024, 65536, 2048); + m_image_buffer = + uint_in_range(*params, "image_buffer", 0, m_image_extent, 0); + m_buffer_size = uint_in_range(*params, "buffer_size", 1, 65536, 10); + m_turdsize = static_cast( + uint_in_range(*params, "turdsize", 0, 65536, m_turdsize)); + + if (get_params().has("img_path")) { + m_image_path = get_params().get_string("img_path"); + } + + if (get_params().has("img_table")) { + m_image_table = get_params().get_string("img_table"); + + for (char const variant : {'i', 'o'}) { + auto const table_name = + fmt::format("{}_{}", m_image_table, variant); + connection->exec(R"( +CREATE TABLE IF NOT EXISTS "{}" ( + type TEXT, + zoom INT4, + x INT4, + y INT4, + rast RASTER +) +)", + table_name); + raster_table_preprocess(table_name); + } + } + + if (get_params().get_bool("make_valid")) { + params->set( + "geom_sql", + "(ST_Dump(ST_CollectionExtract(ST_MakeValid($1), 3))).geom"); + } else { + params->set("geom_sql", "$1"); + } + + if (m_image_extent < 1024U) { + throw std::runtime_error{"width must be at least 1024"}; + } + + if ((m_image_extent & (m_image_extent - 1)) != 0) { + throw std::runtime_error{"width must be power of 2"}; + } + + m_image_buffer = + round_up(static_cast(m_margin * + static_cast(m_image_extent)), + 64U); + m_margin = static_cast(m_image_buffer) / + static_cast(m_image_extent); + + log_gen("Image extent: {}px, buffer: {}px, margin: {}", m_image_extent, + m_image_buffer, m_margin); + + if (with_group_by()) { + dbexec(R"( +PREPARE get_geoms (real, real, real, real) AS + SELECT "{geom_column}", "{group_by_column}" + FROM {src} + WHERE "{geom_column}" && ST_MakeEnvelope($1, $2, $3, $4, 3857) +)"); + dbexec(R"( +PREPARE insert_geoms (geometry, int, int, text) AS + INSERT INTO {dest} ("{geom_column}", x, y, "{group_by_column}") + VALUES ({geom_sql}, $2, $3, $4) +)"); + } else { + dbexec(R"( +PREPARE get_geoms (real, real, real, real) AS + SELECT "{geom_column}", NULL AS param + FROM {src} + WHERE "{geom_column}" && ST_MakeEnvelope($1, $2, $3, $4, 3857) +)"); + dbexec(R"( +PREPARE insert_geoms (geometry, int, int, text) AS + INSERT INTO {dest} ("{geom_column}", x, y) VALUES ({geom_sql}, $2, $3) +)"); + } +} + +static void save_image_to_table(pg_conn_t *connection, canvas_t const &canvas, + tile_t const &tile, double margin, + std::string const ¶m, char const *variant, + std::string const &table_prefix) +{ + auto const wkb = to_hex(canvas.to_wkb(tile, margin)); + + connection->exec("INSERT INTO \"{}_{}\" (type, zoom, x, y, rast)" + " VALUES ('{}', {}, {}, {}, '{}')", + table_prefix, variant, param, tile.zoom(), tile.x(), + tile.y(), wkb); +} + +namespace { + +struct param_canvas_t +{ + canvas_t canvas; + std::size_t points = 0; + + param_canvas_t(unsigned int image_extent, unsigned int image_buffer) + : canvas(image_extent, image_buffer) + {} +}; + +} // anonymous namespace + +using canvas_list_t = std::unordered_map; + +static void draw_from_db(double margin, unsigned int image_extent, + unsigned int image_buffer, canvas_list_t *canvas_list, + pg_conn_t *conn, tile_t const &tile) +{ + auto const box = tile.box(margin); + auto const result = conn->exec_prepared( + "get_geoms", box.min_x(), box.min_y(), box.max_x(), box.max_y()); + + for (int n = 0; n < result.num_tuples(); ++n) { + std::string param = result.get_value(n, 1); + auto const geom = ewkb_to_geom(decode_hex(result.get_value(n, 0))); + + auto const [it, success] = canvas_list->try_emplace( + std::move(param), image_extent, image_buffer); + + it->second.points += it->second.canvas.draw(geom, tile); + } +} + +void gen_tile_raster_union_t::process(tile_t const &tile) +{ + delete_existing(tile); + + canvas_list_t canvas_list; + + log_gen("Read from database and draw polygons..."); + timer(m_timer_draw).start(); + draw_from_db(m_margin, m_image_extent, m_image_buffer, &canvas_list, + &connection(), tile); + timer(m_timer_draw).stop(); + + for (auto &cp : canvas_list) { + auto const ¶m = cp.first; + auto &[canvas, points] = cp.second; + log_gen("Handling param='{}'", param); + + if (!m_image_path.empty()) { + // Save input image for debugging + save_image_to_file(canvas, tile, m_image_path, param, "i", + m_image_extent, m_margin); + } + + if (!m_image_table.empty()) { + // Store input image in database for debugging + save_image_to_table(&connection(), canvas, tile, m_margin, param, + "i", m_image_table); + } + + if (m_buffer_size > 0) { + log_gen("Generalize (buffer={} Mercator units)...", + m_buffer_size * tile.extent() / + static_cast(m_image_extent)); + timer(m_timer_simplify).start(); + canvas.open_close(m_buffer_size); + timer(m_timer_simplify).stop(); + } + + if (!m_image_path.empty()) { + // Save output image for debugging + save_image_to_file(canvas, tile, m_image_path, param, "o", + m_image_extent, m_margin); + } + + if (!m_image_table.empty()) { + // Store output image in database for debugging + save_image_to_table(&connection(), canvas, tile, m_margin, param, + "o", m_image_table); + } + + tracer_t tracer{m_image_extent, m_image_buffer, m_turdsize}; + + log_gen("Vectorize..."); + timer(m_timer_vectorize).start(); + auto const geometries = tracer.trace(canvas, tile); + timer(m_timer_vectorize).stop(); + + log_gen("Reduced from {} points to {} points ({:.1f} %)", points, + tracer.num_points(), + static_cast(tracer.num_points()) / + static_cast(points) * 100); + + log_gen("Write geometries to destination table..."); + timer(m_timer_write).start(); + for (auto const &geom : geometries) { + auto const wkb = geom_to_ewkb(geom); + connection().exec_prepared("insert_geoms", binary_param{wkb}, + tile.x(), tile.y(), param); + } + timer(m_timer_write).stop(); + log_gen("Inserted {} generalized polygons", geometries.size()); + } +} + +void gen_tile_raster_union_t::post() +{ + if (!m_image_table.empty()) { + for (char const variant : {'i', 'o'}) { + raster_table_postprocess( + fmt::format("{}_{}", m_image_table, variant)); + } + } + dbexec("ANALYZE {dest}"); +} diff --git a/src/gen/gen-tile-raster.hpp b/src/gen/gen-tile-raster.hpp new file mode 100644 index 000000000..21d26b6df --- /dev/null +++ b/src/gen/gen-tile-raster.hpp @@ -0,0 +1,49 @@ +#ifndef OSM2PGSQL_GEN_TILE_RASTER_HPP +#define OSM2PGSQL_GEN_TILE_RASTER_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile.hpp" + +#include +#include + +class gen_tile_raster_union_t final : public gen_tile_t +{ +public: + gen_tile_raster_union_t(pg_conn_t *connection, params_t *params); + + ~gen_tile_raster_union_t() override = default; + + void process(tile_t const &tile) override; + + void post() override; + + std::string_view strategy() const noexcept override + { + return "raster-union"; + } + +private: + std::size_t m_timer_draw; + std::size_t m_timer_simplify; + std::size_t m_timer_vectorize; + std::size_t m_timer_write; + + std::string m_image_path; + std::string m_image_table; + double m_margin = 0.0; + std::size_t m_image_extent = 2048; + std::size_t m_image_buffer = 0; + unsigned int m_buffer_size = 10; + int m_turdsize = 2; +}; + +#endif // OSM2PGSQL_GEN_TILE_RASTER_HPP diff --git a/src/gen/gen-tile-vector.cpp b/src/gen/gen-tile-vector.cpp new file mode 100644 index 000000000..62b1e715b --- /dev/null +++ b/src/gen/gen-tile-vector.cpp @@ -0,0 +1,94 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile-vector.hpp" + +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "tile.hpp" + +gen_tile_vector_union_t::gen_tile_vector_union_t(pg_conn_t *connection, + params_t *params) +: gen_tile_t(connection, params), m_timer_simplify(add_timer("simplify")) +{ + if (!get_params().has("margin")) { + params->set("margin", 0.0); + } else { + // We don't need the result, just checking that this is a real number + get_params().get_double("margin"); + } + + if (!get_params().has("buffer_size")) { + params->set("buffer_size", static_cast(10)); + } else { + // We don't need the result, just checking that this is an integer + get_params().get_int64("buffer_size"); + } + + if (with_group_by()) { + dbexec(R"( +PREPARE gen_geoms (int, int, int) AS + WITH gen_tile_input AS ( + SELECT "{group_by_column}" AS col, "{geom_column}" AS geom FROM {src} + WHERE "{geom_column}" && ST_TileEnvelope($1, $2, $3, margin => {margin}) + ), + buffered AS ( + SELECT col, ST_Buffer(geom, {buffer_size}) AS geom + FROM gen_tile_input + ), + merged AS ( + SELECT col, ST_Union(geom) AS geom + FROM buffered GROUP BY col + ), + unbuffered AS ( + SELECT col, ST_Buffer(ST_Buffer(geom, -2 * {buffer_size}), {buffer_size}) AS geom + FROM merged + ) + INSERT INTO {dest} (x, y, "{group_by_column}", "{geom_column}") + SELECT $2, $3, col, (ST_Dump(geom)).geom FROM unbuffered +)"); + } else { + dbexec(R"( +PREPARE gen_geoms (int, int, int) AS + WITH gen_tile_input AS ( + SELECT "{geom_column}" AS geom FROM {src} + WHERE "{geom_column}" && ST_TileEnvelope($1, $2, $3, margin => {margin}) + ), + buffered AS ( + SELECT ST_Buffer(geom, {buffer_size}) AS geom + FROM gen_tile_input + ), + merged AS ( + SELECT ST_Union(geom) AS geom + FROM buffered + ), + unbuffered AS ( + SELECT ST_Buffer(ST_Buffer(geom, -2 * {buffer_size}), {buffer_size}) AS geom + FROM merged + ) + INSERT INTO {dest} (x, y, "{geom_column}") + SELECT $2, $3, (ST_Dump(geom)).geom FROM unbuffered +)"); + } +} + +void gen_tile_vector_union_t::process(tile_t const &tile) +{ + delete_existing(tile); + + log_gen("Generalize..."); + timer(m_timer_simplify).start(); + auto const result = connection().exec_prepared("gen_geoms", tile.zoom(), + tile.x(), tile.y()); + timer(m_timer_simplify).stop(); + log_gen("Inserted {} generalized polygons", result.affected_rows()); +} + +void gen_tile_vector_union_t::post() { dbexec("ANALYZE {dest}"); } diff --git a/src/gen/gen-tile-vector.hpp b/src/gen/gen-tile-vector.hpp new file mode 100644 index 000000000..1aa9867a1 --- /dev/null +++ b/src/gen/gen-tile-vector.hpp @@ -0,0 +1,37 @@ +#ifndef OSM2PGSQL_GEN_TILE_VECTOR_HPP +#define OSM2PGSQL_GEN_TILE_VECTOR_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile.hpp" + +#include + +class gen_tile_vector_union_t final : public gen_tile_t +{ +public: + gen_tile_vector_union_t(pg_conn_t *connection, params_t *params); + + ~gen_tile_vector_union_t() override = default; + + void process(tile_t const &tile) override; + + void post() override; + + std::string_view strategy() const noexcept override + { + return "vector-union"; + } + +private: + std::size_t m_timer_simplify; +}; + +#endif // OSM2PGSQL_GEN_TILE_VECTOR_HPP diff --git a/src/gen/gen-tile.cpp b/src/gen/gen-tile.cpp new file mode 100644 index 000000000..8cc0ed773 --- /dev/null +++ b/src/gen/gen-tile.cpp @@ -0,0 +1,49 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile.hpp" + +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "tile.hpp" + +#include + +gen_tile_t::gen_tile_t(pg_conn_t *connection, params_t *params) +: gen_base_t(connection, params), m_timer_delete(add_timer("delete")) +{ + m_with_group_by = !get_params().get_identifier("group_by_column").empty(); + + if (get_params().get_bool("delete_existing")) { + m_delete_existing = true; + dbexec("PREPARE del_geoms (int, int) AS" + " DELETE FROM {dest} WHERE x=$1 AND y=$2"); + } +} + +void gen_tile_t::delete_existing(tile_t const &tile) +{ + if (!m_delete_existing) { + return; + } + + if (debug()) { + log_gen("Delete geometries from destination table..."); + } + + timer(m_timer_delete).start(); + auto const result = + connection().exec_prepared("del_geoms", tile.x(), tile.y()); + timer(m_timer_delete).stop(); + + if (debug()) { + log_gen("Deleted {} rows.", result.affected_rows()); + } +} diff --git a/src/gen/gen-tile.hpp b/src/gen/gen-tile.hpp new file mode 100644 index 000000000..626fbf161 --- /dev/null +++ b/src/gen/gen-tile.hpp @@ -0,0 +1,36 @@ +#ifndef OSM2PGSQL_GEN_TILE_HPP +#define OSM2PGSQL_GEN_TILE_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-base.hpp" + +/** + * Base class for generalizations based on tiles. + */ +class gen_tile_t : public gen_base_t +{ +public: + bool on_tiles() const noexcept override { return true; } + +protected: + gen_tile_t(pg_conn_t *connection, params_t *params); + + void delete_existing(tile_t const &tile); + + bool with_group_by() const noexcept { return m_with_group_by; } + +private: + std::size_t m_timer_delete; + bool m_delete_existing = false; + bool m_with_group_by = false; +}; + +#endif // OSM2PGSQL_GEN_TILE_HPP diff --git a/src/gen/osm2pgsql-gen.cpp b/src/gen/osm2pgsql-gen.cpp new file mode 100644 index 000000000..353557fc1 --- /dev/null +++ b/src/gen/osm2pgsql-gen.cpp @@ -0,0 +1,683 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +/** + * \file + * + * This program is used for accessing generalization functionality. It is + * experimental and might or might not be integrated into osm2pgsql itself + * in the future. + */ + +#include "canvas.hpp" +#include "debug-output.hpp" +#include "expire-output.hpp" +#include "flex-lua-expire-output.hpp" +#include "flex-lua-geom.hpp" +#include "flex-lua-table.hpp" +#include "flex-table.hpp" +#include "format.hpp" +#include "gen-base.hpp" +#include "gen-create.hpp" +#include "logging.hpp" +#include "lua-init.hpp" +#include "lua-setup.hpp" +#include "lua-utils.hpp" +#include "options.hpp" +#include "params.hpp" +#include "pgsql-capabilities.hpp" +#include "pgsql.hpp" +#include "tile.hpp" +#include "util.hpp" +#include "version.hpp" + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr std::size_t const max_force_single_thread = 4; + +// Lua can't call functions on C++ objects directly. This macro defines simple +// C "trampoline" functions which are called from Lua which get the current +// context (the genproc_t object) and call the respective function on the +// context object. +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define TRAMPOLINE(func_name, lua_name) \ + static int lua_trampoline_##func_name(lua_State *lua_state) \ + { \ + try { \ + return static_cast(luaX_get_context(lua_state)) \ + ->func_name(); \ + } catch (std::exception const &e) { \ + return luaL_error(lua_state, "Error in '" #lua_name "': %s\n", \ + e.what()); \ + } catch (...) { \ + return luaL_error(lua_state, \ + "Unknown error in '" #lua_name "'.\n"); \ + } \ + } + +static void show_help() +{ + fmt::print(R"(osm2pgsql-gen [OPTIONS] +Generalization of OSM data. + +This program is EXPERIMENTAL and might change without notice. + +Options: + -h|--help Print this help text and stop + -a|--append Run in append mode + -c|--create Run in create mode (default) + -j|--jobs Number of parallel jobs (default 1) + -l|--log-level=LEVEL Log level (debug, info (default), warn, error) + --log-sql Log SQL commands + --cimg-info Call info() function of CImg library + +Database options: + -d|--database=DB The name of the PostgreSQL database to connect to or + a PostgreSQL conninfo string. + -U|--username=NAME PostgreSQL user name. + -W|--password Force password prompt. + -H|--host=HOST Database server host name or socket location. + -P|--port=PORT Database server port. +)"); +} + +static char const *const short_options = "acd:hH:j:l:P:S:U:W"; + +static std::array const long_options = { + {{"help", no_argument, nullptr, 'h'}, + {"append", no_argument, nullptr, 'a'}, + {"create", no_argument, nullptr, 'c'}, + {"jobs", required_argument, nullptr, 'j'}, + {"database", required_argument, nullptr, 'd'}, + {"user", required_argument, nullptr, 'U'}, + {"host", required_argument, nullptr, 'H'}, + {"port", required_argument, nullptr, 'P'}, + {"password", no_argument, nullptr, 'W'}, + {"log-level", required_argument, nullptr, 'l'}, + {"style", required_argument, nullptr, 'S'}, + {"cimg-info", no_argument, nullptr, 200}, + {"log-sql", no_argument, nullptr, 201}, + {nullptr, 0, nullptr, 0}}}; + +struct tile_extent +{ + uint32_t xmin = 0; + uint32_t ymin = 0; + uint32_t xmax = 0; + uint32_t ymax = 0; + bool valid = false; +}; + +static bool table_is_empty(pg_conn_t const &db_connection, + std::string const &schema, std::string const &table) +{ + auto const result = db_connection.exec("SELECT 1 FROM {} LIMIT 1", + qualified_name(schema, table)); + return result.num_tuples() == 0; +} + +static tile_extent get_extent_from_db(pg_conn_t const &db_connection, + std::string const &schema, + std::string const &table, + std::string const &column, uint32_t zoom) +{ + if (table_is_empty(db_connection, schema, table)) { + return {}; + } + + auto const result = db_connection.exec( + "SELECT ST_XMin(e), ST_YMin(e), ST_XMax(e), ST_YMax(e)" + " FROM ST_EstimatedExtent('{}', '{}', '{}') AS e", + schema, table, column); + + if (result.num_tuples() == 0 || result.is_null(0, 0)) { + return {}; + } + + double const extent_xmin = strtod(result.get_value(0, 0), nullptr); + double const extent_ymin = strtod(result.get_value(0, 1), nullptr); + double const extent_xmax = strtod(result.get_value(0, 2), nullptr); + double const extent_ymax = strtod(result.get_value(0, 3), nullptr); + log_debug("Extent: ({} {}, {} {})", extent_xmin, extent_ymin, extent_xmax, + extent_ymax); + + return {osmium::geom::mercx_to_tilex(zoom, extent_xmin), + osmium::geom::mercy_to_tiley(zoom, extent_ymax), + osmium::geom::mercx_to_tilex(zoom, extent_xmax), + osmium::geom::mercy_to_tiley(zoom, extent_ymin), true}; +} + +static tile_extent get_extent_from_db(pg_conn_t const &db_connection, + params_t const ¶ms, uint32_t zoom) +{ + auto const schema = params.get_string("schema", "public"); + std::string table; + if (params.has("src_table")) { + table = params.get_string("src_table"); + } else if (params.has("src_tables")) { + table = params.get_string("src_tables"); + auto const n = table.find(','); + if (n != std::string::npos) { + table.resize(n); + } + } else { + throw std::runtime_error{"Need 'src_table' or 'src_tables' param."}; + } + auto const geom_column = params.get_string("geom_column", "geom"); + return get_extent_from_db(db_connection, schema, table, geom_column, zoom); +} + +static std::vector> +get_tiles_from_table(pg_conn_t const &connection, std::string const &table) +{ + std::vector> tiles; + + auto const result = connection.exec(R"(SELECT x, y FROM "{}")", table); + + for (int n = 0; n < result.num_tuples(); ++n) { + char *end = nullptr; + auto const x = std::strtoul(result.get_value(n, 0), &end, 10); + auto const y = std::strtoul(result.get_value(n, 1), &end, 10); + tiles.emplace_back(x, y); + } + + return tiles; +} + +static uint32_t parse_uint32(param_value_t const &val, char const *context, + uint32_t default_value) +{ + if (std::holds_alternative(val)) { + return default_value; + } + if (!std::holds_alternative(val)) { + throw fmt_error("Invalid value for {}.", context); + } + return static_cast(std::get(val)); +} + +class tile_processor_t +{ +public: + tile_processor_t(gen_base_t *generalizer, std::size_t num_tiles) + : m_generalizer(generalizer), m_num_tiles(num_tiles) + {} + + void operator()(tile_t const &tile) + { + log_debug("Processing tile {}/{}/{} ({} of {})...", tile.zoom(), + tile.x(), tile.y(), ++m_count, m_num_tiles); + m_generalizer->process(tile); + } + +private: + gen_base_t *m_generalizer; + std::size_t m_count = 0; + std::size_t m_num_tiles; +}; + +void run_tile_gen(std::string const &conninfo, gen_base_t *master_generalizer, + params_t params, uint32_t zoom, + std::vector> *queue, + std::mutex *mut, unsigned int n) +{ + get_logger().init_thread(n); + + log_debug("Started generalizer thread for '{}'.", + master_generalizer->strategy()); + pg_conn_t db_connection{conninfo}; + std::string const strategy{master_generalizer->strategy()}; + auto generalizer = create_generalizer(strategy, &db_connection, ¶ms); + + while (true) { + std::pair p; + { + std::lock_guard const guard{*mut}; + if (queue->empty()) { + master_generalizer->merge_timers(*generalizer); + break; + } + p = queue->back(); + queue->pop_back(); + } + + generalizer->process({zoom, p.first, p.second}); + } + log_debug("Shutting down generalizer thread."); +} + +class genproc_t +{ +public: + genproc_t(std::string const &filename, std::string conninfo, bool append, + uint32_t jobs); + + int app_define_table() + { +#if 0 + if (m_calling_context != calling_context::main) { + throw std::runtime_error{ + "Database tables have to be defined in the" + " main Lua code, not in any of the callbacks."}; + } +#endif + + return setup_flex_table(m_lua_state.get(), &m_tables, &m_expire_outputs, + true, m_append); + } + + int app_define_expire_output() + { + return setup_flex_expire_output(m_lua_state.get(), &m_expire_outputs); + } + + int app_run_gen() + { + log_debug("Running configured generalizer (run {})...", ++m_gen_run); + + if (lua_type(lua_state(), 1) != LUA_TSTRING) { + throw std::runtime_error{"Argument #1 to 'run_gen' must be a " + "string naming the strategy."}; + } + + std::string const strategy = lua_tostring(lua_state(), 1); + log_debug("Generalizer strategy '{}'", strategy); + + if (lua_type(lua_state(), 2) != LUA_TTABLE) { + throw std::runtime_error{"Argument #2 to 'run_gen' must be a " + "table with parameters."}; + } + + auto params = parse_params(); + + write_to_debug_log(params, "Params (config):"); + + log_debug("Connecting to database..."); + pg_conn_t db_connection{m_conninfo}; + + log_debug("Creating generalizer..."); + auto generalizer = + create_generalizer(strategy, &db_connection, ¶ms); + + log_debug("Generalizer '{}' ({}) initialized.", generalizer->name(), + generalizer->strategy()); + + if (m_append) { + params.set("delete_existing", true); + } + + write_to_debug_log(params, "Params (after initialization):"); + + if (generalizer->on_tiles()) { + process_tiles(db_connection, params, generalizer.get()); + } else { + generalizer->process(); + } + + log_debug("Running generalizer postprocessing..."); + generalizer->post(); + + log_debug("Generalizer processing done."); + + log_debug("Timers:"); + for (auto const &timer : generalizer->timers()) { + log_debug(fmt::format( + " {:10} {:>10L}", timer.name() + ":", + std::chrono::duration_cast( + timer.elapsed()))); + } + log_debug("Finished generalizer '{}' (run {}).", generalizer->name(), + m_gen_run); + + return 0; + } + + int app_run_sql() + { + if (lua_type(lua_state(), 1) != LUA_TTABLE) { + throw std::runtime_error{"Argument #1 to 'run_sql' must be a " + "table with parameters."}; + } + + std::string const description = + luaX_get_table_string(lua_state(), "description", 1, "Argument #1"); + std::string const sql = + luaX_get_table_string(lua_state(), "sql", 1, "Argument #1"); + + log_debug("Running SQL command: {}.", description); + + util::timer_t timer_sql; + pg_conn_t const db_connection{m_conninfo}; + db_connection.exec(sql); + log_info("SQL command took {}.", + util::human_readable_duration(timer_sql.stop())); + + return 0; + } + + void run(); + +private: + params_t parse_params() + { + params_t params; + + lua_pushnil(lua_state()); + while (lua_next(lua_state(), 2) != 0) { + if (lua_type(lua_state(), -2) != LUA_TSTRING) { + throw std::runtime_error{"Argument #2 must have string keys"}; + } + auto const *key = lua_tostring(lua_state(), -2); + + switch (lua_type(lua_state(), -1)) { + case LUA_TSTRING: + params.set(key, lua_tostring(lua_state(), -1)); + break; + case LUA_TNUMBER: +#if LUA_VERSION_NUM >= 503 + if (lua_isinteger(lua_state(), -1)) { + params.set(key, static_cast( + lua_tointeger(lua_state(), -1))); + } else { + params.set(key, static_cast( + lua_tonumber(lua_state(), -1))); + } +#else + params.set(key, + static_cast(lua_tonumber(lua_state(), -1))); +#endif + break; + case LUA_TBOOLEAN: + params.set(key, + static_cast(lua_toboolean(lua_state(), -1))); + break; + case LUA_TNIL: + break; + default: + throw std::runtime_error{"Argument #2 must have string values"}; + } + + lua_pop(lua_state(), 1); + } + return params; + } + + void process_tiles(pg_conn_t const &db_connection, params_t const ¶ms, + gen_base_t *generalizer) + { + uint32_t const zoom = parse_uint32(params.get("zoom"), "zoom", 0); + std::vector> tile_list; + if (m_append) { + auto const table = params.get_string("expire_list"); + log_debug("Running generalizer for expire list from table '{}'...", + table); + tile_list = get_tiles_from_table(db_connection, table); + log_debug("Truncating table '{}'...", table); + db_connection.exec("TRUNCATE {}", table); + } else { + auto const extent = get_extent_from_db(db_connection, params, zoom); + + if (extent.valid) { + auto const num_tiles = (extent.xmax - extent.xmin + 1) * + (extent.ymax - extent.ymin + 1); + log_debug("Running generalizer for bounding box x{}-{}, y{}-{}" + " on zoom={}...", + extent.xmin, extent.xmax, extent.ymin, extent.ymax, + zoom); + tile_list.reserve(num_tiles); + for (unsigned x = extent.xmin; x <= extent.xmax; ++x) { + for (unsigned y = extent.ymin; y <= extent.ymax; ++y) { + tile_list.emplace_back(x, y); + } + } + } else { + log_debug("Source table empty, nothing to do."); + } + } + log_debug("Need to process {} tiles.", tile_list.size()); + if (m_jobs == 1 || tile_list.size() < max_force_single_thread) { + log_debug("Running in single-threaded mode."); + tile_processor_t tp{generalizer, tile_list.size()}; + while (!tile_list.empty()) { + auto [x, y] = tile_list.back(); + tp({zoom, x, y}); + tile_list.pop_back(); + } + } else { + log_debug("Running in multi-threaded mode."); + std::mutex mut; + std::vector threads; + for (unsigned int n = 1; n <= m_jobs; ++n) { + threads.emplace_back(run_tile_gen, m_conninfo, generalizer, + params, zoom, &tile_list, &mut, n); + } + for (auto &t : threads) { + t.join(); + } + } + } + + lua_State *lua_state() const noexcept { return m_lua_state.get(); } + + std::shared_ptr m_lua_state{ + luaL_newstate(), [](lua_State *state) { lua_close(state); }}; + + std::vector m_tables; + std::vector m_expire_outputs; + + std::string m_conninfo; + std::size_t m_gen_run = 0; + uint32_t m_jobs; + bool m_append; +}; // class genproc_t + +TRAMPOLINE(app_define_table, define_table) +TRAMPOLINE(app_define_expire_output, define_expire_output) +TRAMPOLINE(app_run_gen, run_gen) +TRAMPOLINE(app_run_sql, run_sql) + +genproc_t::genproc_t(std::string const &filename, std::string conninfo, + bool append, uint32_t jobs) +: m_conninfo(std::move(conninfo)), m_jobs(jobs), m_append(append) +{ + setup_lua_environment(lua_state(), filename, append); + + luaX_add_table_func(lua_state(), "define_table", + lua_trampoline_app_define_table); + luaX_add_table_func(lua_state(), "define_expire_output", + lua_trampoline_app_define_expire_output); + + luaX_add_table_func(lua_state(), "run_gen", lua_trampoline_app_run_gen); + luaX_add_table_func(lua_state(), "run_sql", lua_trampoline_app_run_sql); + + init_geometry_class(lua_state()); + + // Load compiled in init.lua + if (luaL_dostring(lua_state(), lua_init())) { + throw fmt_error("Internal error in Lua setup: {}.", + lua_tostring(lua_state(), -1)); + } + + // Load user config file + luaX_set_context(lua_state(), this); + if (luaL_dofile(lua_state(), filename.c_str())) { + throw fmt_error("Error loading lua config: {}.", + lua_tostring(lua_state(), -1)); + } + + write_expire_output_list_to_debug_log(m_expire_outputs); + write_table_list_to_debug_log(m_tables); +} + +void genproc_t::run() +{ + lua_getglobal(lua_state(), "osm2pgsql"); + lua_getfield(lua_state(), -1, "process_gen"); + + if (lua_isnil(lua_state(), -1)) { + log_warn("No function 'osm2pgsql.process_gen()'. Nothing to do."); + return; + } + + if (luaX_pcall(lua_state(), 0, 0)) { + throw fmt_error( + "Failed to execute Lua function 'osm2pgsql.process_gen': {}.", + lua_tostring(lua_state(), -1)); + } +} + +int main(int argc, char *argv[]) +{ + try { + database_options_t database_options; + std::string log_level; + std::string style; + uint32_t jobs = 1; + bool pass_prompt = false; + bool append = false; + + int c = 0; + // NOLINTNEXTLINE(concurrency-mt-unsafe) + while (-1 != (c = getopt_long(argc, argv, short_options, + long_options.data(), nullptr))) { + switch (c) { + case 'h': + show_help(); + return 0; + case 'a': // --append + append = true; + break; + case 'c': // --create + append = false; + break; + case 'j': // --jons + jobs = + std::clamp(std::strtoul(optarg, nullptr, 10), 1UL, 256UL); + break; + case 'd': // --database + database_options.db = optarg; + break; + case 'U': // --username + database_options.username = optarg; + break; + case 'W': // --password + pass_prompt = true; + break; + case 'H': // --host + database_options.host = optarg; + break; + case 'P': // --port + database_options.port = optarg; + break; + case 'l': + log_level = optarg; + break; + case 'S': + style = optarg; + break; + case 200: + canvas_t::info(); + return 0; + case 201: + get_logger().enable_sql(); + break; + default: + log_error("Unknown argument"); + return 2; + } + } + + if (log_level == "debug") { + get_logger().set_level(log_level::debug); + } else if (log_level == "info" || log_level.empty()) { + get_logger().set_level(log_level::info); + } else if (log_level == "warn") { + get_logger().set_level(log_level::warn); + } else if (log_level == "error") { + get_logger().set_level(log_level::error); + } else { + log_error("Unknown log level: {}. " + "Use 'debug', 'info', 'warn', or 'error'.", + log_level); + return 2; + } + + if (style.empty()) { + log_error("Need --style/-S option"); + return 2; + } + + if (jobs < 1 || jobs > 32) { + log_error("The --jobs/-j parameter must be between 1 and 32."); + return 2; + } + + util::timer_t timer_overall; + + log_info("osm2pgsql-gen version {}", get_osm2pgsql_version()); + log_warn("This is an EXPERIMENTAL extension to osm2pgsql."); + + if (append) { + log_debug("Running in append mode."); + } else { + log_debug("Running in create mode."); + } + + if (jobs == 1) { + log_debug("Running in single-threaded mode."); + } else { + log_debug( + "Running in multi-threaded mode with a maximum of {} threads.", + jobs); + } + + if (pass_prompt) { + database_options.password = util::get_password(); + } + auto const conninfo = build_conninfo(database_options); + + log_debug("Checking database capabilities..."); + { + pg_conn_t const db_connection{conninfo}; + init_database_capabilities(db_connection); + } + + genproc_t gen{style, conninfo, append, jobs}; + gen.run(); + + osmium::MemoryUsage const mem; + log_info("Memory: {}MB current, {}MB peak", mem.current(), mem.peak()); + + log_info("osm2pgsql-gen took {} overall.", + util::human_readable_duration(timer_overall.stop())); + } catch (std::exception const &e) { + log_error("{}", e.what()); + return 1; + } catch (...) { + log_error("Unknown exception."); + return 1; + } + + return 0; +} diff --git a/src/gen/params.cpp b/src/gen/params.cpp new file mode 100644 index 000000000..b50bae946 --- /dev/null +++ b/src/gen/params.cpp @@ -0,0 +1,125 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "params.hpp" + +#include "format.hpp" +#include "logging.hpp" +#include "overloaded.hpp" +#include "pgsql.hpp" + +std::string to_string(param_value_t const &value) +{ + return std::visit( + overloaded{[](null_param_t) { return std::string{}; }, + [](std::string val) { return val; }, + [](auto const &val) { return fmt::to_string(val); }}, + value); +} + +param_value_t params_t::get(std::string const &key) const +{ + return m_map.at(key); +} + +bool params_t::has(std::string const &key) const noexcept +{ + return m_map.count(key) > 0; +} + +bool params_t::get_bool(std::string const &key, bool default_value) const +{ + return get_by_type(key, default_value); +} + +int64_t params_t::get_int64(std::string const &key, int64_t default_value) const +{ + return get_by_type(key, default_value); +} + +double params_t::get_double(std::string const &key, double default_value) const +{ + auto const it = m_map.find(key); + if (it == m_map.end()) { + return default_value; + } + + if (std::holds_alternative(it->second)) { + return std::get(it->second); + } + + if (std::holds_alternative(it->second)) { + return static_cast(std::get(it->second)); + } + + throw fmt_error("Invalid value '{}' for {}.", to_string(it->second), key); +} + +std::string params_t::get_string(std::string const &key) const +{ + auto const it = m_map.find(key); + if (it == m_map.end()) { + throw fmt_error("Missing parameter '{}' on generalizer.", key); + } + return to_string(it->second); +} + +std::string params_t::get_string(std::string const &key, + std::string const &default_value) const +{ + return get_by_type(key, default_value); +} + +std::string params_t::get_identifier(std::string const &key) const +{ + auto const it = m_map.find(key); + if (it == m_map.end()) { + return {}; + } + std::string result = to_string(it->second); + check_identifier(result, key.c_str()); + return result; +} + +void params_t::check_identifier_with_default(std::string const &key, + std::string default_value) +{ + auto const it = m_map.find(key); + if (it == m_map.end()) { + m_map.emplace(key, std::move(default_value)); + } else { + check_identifier(to_string(it->second), key.c_str()); + } +} + +unsigned int uint_in_range(params_t const ¶ms, std::string const &key, + unsigned int min, unsigned int max, + unsigned int default_value) +{ + int64_t const value = params.get_int64(key, default_value); + if (value < 0 || value > std::numeric_limits::max()) { + throw fmt_error("Invalid value '{}' for {}.", value, key); + } + auto uvalue = static_cast(value); + if (uvalue < min || uvalue > max) { + throw fmt_error("Invalid value '{}' for {}.", value, key); + } + return uvalue; +} + +void write_to_debug_log(params_t const ¶ms, char const *message) +{ + if (!get_logger().debug_enabled()) { + return; + } + log_debug(message); + for (auto const &[key, value] : params) { + log_debug(" {}={}", key, to_string(value)); + } +} diff --git a/src/gen/params.hpp b/src/gen/params.hpp new file mode 100644 index 000000000..f0f766f81 --- /dev/null +++ b/src/gen/params.hpp @@ -0,0 +1,96 @@ +#ifndef OSM2PGSQL_PARAMS_HPP +#define OSM2PGSQL_PARAMS_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "logging.hpp" + +#include +#include +#include + +/// A "NULL" value for a parameter. Same as not set. +using null_param_t = std::monostate; + +/// A parameter value can have one of several types. +using param_value_t = + std::variant; + +/// Convert a parameter value into a string. +std::string to_string(param_value_t const &value); + +/** + * A collection of parameters. + */ +class params_t +{ +public: + template + void set(K &&key, V &&value) + { + m_map.insert_or_assign(std::forward(key), std::forward(value)); + } + + template + void remove(K &&key) + { + m_map.erase(std::forward(key)); + } + + bool has(std::string const &key) const noexcept; + + param_value_t get(std::string const &key) const; + + bool get_bool(std::string const &key, bool default_value = false) const; + + int64_t get_int64(std::string const &key, int64_t default_value = 0) const; + + double get_double(std::string const &key, double default_value = 0.0) const; + + std::string get_string(std::string const &key) const; + + std::string get_string(std::string const &key, + std::string const &default_value) const; + + std::string get_identifier(std::string const &key) const; + + void check_identifier_with_default(std::string const &key, + std::string default_value); + + auto begin() const noexcept { return m_map.begin(); } + + auto end() const noexcept { return m_map.end(); } + +private: + template + T get_by_type(std::string const &key, T default_value) const + { + auto const it = m_map.find(key); + if (it == m_map.end()) { + return default_value; + } + + if (!std::holds_alternative(it->second)) { + throw fmt_error("Invalid value '{}' for {}.", to_string(it->second), + key); + } + return std::get(it->second); + } + + std::map m_map; +}; // class params_t + +void write_to_debug_log(params_t const ¶ms, char const *message); + +unsigned int uint_in_range(params_t const ¶ms, std::string const &key, + unsigned int min, unsigned int max, + unsigned int default_value); + +#endif // OSM2PGSQL_PARAMS_HPP diff --git a/src/gen/raster.cpp b/src/gen/raster.cpp new file mode 100644 index 000000000..e552c7cd2 --- /dev/null +++ b/src/gen/raster.cpp @@ -0,0 +1,66 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "raster.hpp" + +#include "canvas.hpp" +#include "format.hpp" +#include "pgsql.hpp" +#include "tile.hpp" + +#include + +template +void append(std::string *str, T value) +{ + str->append(reinterpret_cast(&value), sizeof(T)); +} + +void add_raster_header(std::string *wkb, wkb_raster_header const &data) +{ + append(wkb, data.endianness); + append(wkb, data.version); + append(wkb, data.nBands); + append(wkb, data.scaleX); + append(wkb, data.scaleY); + append(wkb, data.ipX); + append(wkb, data.ipY); + append(wkb, data.skewX); + append(wkb, data.skewY); + append(wkb, data.srid); + append(wkb, data.width); + append(wkb, data.height); +} + +void add_raster_band(std::string *wkb, wkb_raster_band const &data) +{ + append(wkb, data.bits); + append(wkb, data.nodata); +} + +void save_image_to_file(canvas_t const &canvas, tile_t const &tile, + std::string const &path, std::string const ¶m, + char const *variant, unsigned int image_extent, + double margin) +{ + std::string name{fmt::format("{}-{}-{}-{}{}{}.", path, tile.x(), tile.y(), + param, param.empty() ? "" : "-", variant)}; + + // write image file + canvas.save(name + "png"); + + // write world file + auto const pixel_size = tile.extent() / image_extent; + name += "wld"; + auto *file = std::fopen(name.c_str(), "w"); + fmt::print(file, "{0}\n0.0\n0.0\n-{0}\n{1}\n{2}\n", pixel_size, + tile.xmin(margin) + pixel_size / 2, + tile.ymax(margin) - pixel_size / 2); + (void)std::fclose(file); +} diff --git a/src/gen/raster.hpp b/src/gen/raster.hpp new file mode 100644 index 000000000..82a86ea0b --- /dev/null +++ b/src/gen/raster.hpp @@ -0,0 +1,64 @@ +#ifndef OSM2PGSQL_RASTER_HPP +#define OSM2PGSQL_RASTER_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include +#include + +class canvas_t; +class pg_conn_t; +class tile_t; + +/** + * \file + * + * Helper functions for creating raster images in PostgreSQL/PostGIS. + * https://trac.osgeo.org/postgis/wiki/WKTRaster/RFC/RFC2_V0WKBFormat + */ + +struct wkb_raster_header +{ + uint8_t endianness = +#if __BYTE_ORDER == __LITTLE_ENDIAN + 1 // Little Endian +#else + 0 // Big Endian +#endif + ; + uint16_t version = 0; + uint16_t nBands = 0; + double scaleX = 0.0; + double scaleY = 0.0; + double ipX = 0.0; + double ipY = 0.0; + double skewX = 0.0; + double skewY = 0.0; + int32_t srid = 3857; + uint16_t width = 0; + uint16_t height = 0; +}; + +struct wkb_raster_band +{ + uint8_t bits = 0; + uint8_t nodata = 0; +}; + +void add_raster_header(std::string *wkb, wkb_raster_header const &data); + +void add_raster_band(std::string *wkb, wkb_raster_band const &data); + +void save_image_to_file(canvas_t const &canvas, tile_t const &tile, + std::string const &path, std::string const ¶m, + char const *variant, unsigned int image_extent, + double margin); + +#endif // OSM2PGSQL_RASTER_HPP diff --git a/src/gen/tracer.cpp b/src/gen/tracer.cpp new file mode 100644 index 000000000..5321c3eae --- /dev/null +++ b/src/gen/tracer.cpp @@ -0,0 +1,108 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "tracer.hpp" + +#include "geom-boost-adaptor.hpp" + +#include +#include +#include + +geom::point_t tracer_t::make_point(potrace_dpoint_t const &p) const noexcept +{ + return {p.x - static_cast(m_buffer), + static_cast(m_extent + m_buffer) - p.y}; +} + +std::vector +tracer_t::trace(canvas_t const &canvas, tile_t const &tile, double min_area) +{ + prepare(canvas); + + m_state.reset(potrace_trace(m_param.get(), &m_bitmap)); + if (!m_state || m_state->status != POTRACE_STATUS_OK) { + throw std::runtime_error{"potrace failed"}; + } + + return build_geometries(tile, m_state->plist, min_area); +} + +void tracer_t::reset() +{ + m_bits.clear(); + m_state.reset(); + m_num_points = 0; +} + +void tracer_t::prepare(canvas_t const &canvas) noexcept +{ + std::size_t const size = canvas.size(); + assert(size % bits_per_word == 0); + + m_bits.reserve((size * size) / bits_per_word); + + unsigned char const *d = canvas.begin(); + while (d != canvas.end()) { + potrace_word w = 0x1U & *d++; + for (std::size_t n = 1; n < bits_per_word; ++n) { + w <<= 1U; + assert(d != canvas.end()); + w |= 0x1U & *d++; + } + m_bits.push_back(w); + } + + m_bitmap = {int(size), int(size), int(size / bits_per_word), m_bits.data()}; +} + +std::vector +tracer_t::build_geometries(tile_t const &tile, potrace_path_t const *plist, + double min_area) noexcept +{ + std::vector geometries; + if (!plist) { + return geometries; + } + + for (potrace_path_t const *path = plist; path != nullptr; + path = path->next) { + + geom::ring_t ring; + + auto const n = path->curve.n; + assert(path->curve.tag[n - 1] == POTRACE_CORNER); + ring.push_back(tile.to_world_coords(make_point(path->curve.c[n - 1][2]), + m_extent)); + for (int i = 0; i < n; ++i) { + assert(path->curve.tag[i] == POTRACE_CORNER); + auto const &c = path->curve.c[i]; + ring.push_back(tile.to_world_coords(make_point(c[1]), m_extent)); + ring.push_back(tile.to_world_coords(make_point(c[2]), m_extent)); + } + + auto const ring_area = + std::abs(static_cast(boost::geometry::area(ring))); + if (ring_area >= min_area) { + m_num_points += ring.size(); + + if (path->sign == '+') { + geometries.emplace_back(geom::polygon_t{}, 3857) + .get() + .outer() = std::move(ring); + } else { + assert(!geometries.empty()); + geometries.back().get().add_inner_ring( + std::move(ring)); + } + } + } + + return geometries; +} diff --git a/src/gen/tracer.hpp b/src/gen/tracer.hpp new file mode 100644 index 000000000..6026070e6 --- /dev/null +++ b/src/gen/tracer.hpp @@ -0,0 +1,76 @@ +#ifndef OSM2PGSQL_TRACER_HPP +#define OSM2PGSQL_TRACER_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "canvas.hpp" +#include "geom.hpp" +#include "tile.hpp" + +#include + +#include +#include + +class tracer_t +{ +public: + tracer_t(std::size_t extent, std::size_t buffer, int turdsize) + : m_param(potrace_param_default()), m_extent(extent), m_buffer(buffer) + { + m_param->alphamax = 0.0; + m_param->turdsize = turdsize; + } + + std::vector + trace(canvas_t const &canvas, tile_t const &tile, double min_area = 0.0); + + void reset(); + + std::size_t num_points() const noexcept { return m_num_points; } + +private: + static constexpr auto const bits_per_word = sizeof(potrace_word) * 8; + + geom::point_t make_point(potrace_dpoint_t const &p) const noexcept; + + struct potrace_param_deleter + { + void operator()(potrace_param_t *ptr) const noexcept + { + potrace_param_free(ptr); + } + }; + + struct potrace_state_deleter + { + void operator()(potrace_state_t *ptr) const noexcept + { + potrace_state_free(ptr); + } + }; + + void prepare(canvas_t const &canvas) noexcept; + + std::vector build_geometries(tile_t const &tile, + potrace_path_t const *plist, + double min_area) noexcept; + + std::vector m_bits; + potrace_bitmap_t m_bitmap{}; + std::unique_ptr m_param; + std::unique_ptr m_state; + std::size_t m_extent; + std::size_t m_buffer; + std::size_t m_num_points = 0; + +}; // class tracer_t + +#endif // OSM2PGSQL_TRACER_HPP diff --git a/src/pgsql.hpp b/src/pgsql.hpp index 782d79ae8..3736a115b 100644 --- a/src/pgsql.hpp +++ b/src/pgsql.hpp @@ -126,6 +126,20 @@ class pg_result_t std::unique_ptr m_result; }; +/** + * Wrapper class for query parameters that should be sent to the database + * as binary parameter. + */ +class binary_param : public std::string_view +{ +public: + using std::string_view::string_view; + + binary_param(std::string const &str) + : std::string_view(str.data(), str.size()) + {} +}; + /** * PostgreSQL connection. * @@ -229,6 +243,8 @@ class pg_conn_t return 0; } else if constexpr (std::is_same_v) { return 0; + } else if constexpr (std::is_same_v) { + return 0; } return 1; } @@ -240,12 +256,18 @@ class pg_conn_t * strings. */ template - static char const *to_str(std::vector *data, T const ¶m) + static char const *to_str(std::vector *data, int *length, + int *bin, T const ¶m) { if constexpr (std::is_same_v) { return param; } else if constexpr (std::is_same_v) { + *length = param.size(); return param.c_str(); + } else if constexpr (std::is_same_v) { + *length = param.size(); + *bin = 1; + return param.data(); } return data->emplace_back(fmt::to_string(param)).c_str(); } @@ -275,16 +297,21 @@ class pg_conn_t std::vector exec_params; exec_params.reserve(total_buffers_needed); + std::array lengths = {0}; + std::array bins = {0}; + // This array holds the pointers to all parameter strings, either // to the original string parameters or to the recently converted // in the exec_params vector. + std::size_t n = 0; + std::size_t m = 0; std::array param_ptrs = { - to_str>(&exec_params, + to_str>(&exec_params, &lengths[n++], &bins[m++], std::forward(params))...}; return exec_prepared_internal(stmt, sizeof...(params), - param_ptrs.data(), nullptr, nullptr, - result_as_binary ? 1 : 0); + param_ptrs.data(), lengths.data(), + bins.data(), result_as_binary ? 1 : 0); } struct pg_conn_deleter_t diff --git a/tests/test-pgsql.cpp b/tests/test-pgsql.cpp index f394fc224..4d9806ed4 100644 --- a/tests/test-pgsql.cpp +++ b/tests/test-pgsql.cpp @@ -56,18 +56,6 @@ TEST_CASE("exec with invalid SQL should fail") REQUIRE_THROWS(conn.exec("XYZ")); } -TEST_CASE("exec_prepared without parameters should work") -{ - auto const conn = db.db().connect(); - conn.exec("PREPARE test AS SELECT 42"); - - auto const result = conn.exec_prepared("test"); - REQUIRE(result.status() == PGRES_TUPLES_OK); - REQUIRE(result.num_fields() == 1); - REQUIRE(result.num_tuples() == 1); - REQUIRE(result.get(0, 0) == "42"); -} - TEST_CASE("exec_prepared with single string parameters should work") { auto const conn = db.db().connect();