diff --git a/.DS_Store b/.DS_Store index fcd267b..426e60a 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index 10361e8..c4d6564 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ mdbdata/ wikidata/ .DS_Store +maps-data/maptiler-osm-2021-05-31-v3.12.1-planet.mbtiles +.DS_Store diff --git a/README.md b/README.md index 785925a..ddc232d 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,13 @@ ## Install Instructions -$ sudo docker-compose up -d \ No newline at end of file +$ sudo docker-compose up -d + +## Setup and use our own Tile Server +This is needed to use maps in extensions. + +One way is to use the Tile Server Docker from here: +https://hub.docker.com/r/klokantech/openmaptiles-server + +To start use this command: + +docker run --rm -it -v ./maps_data:/data -p 8091:80 klokantech/openmaptiles-server \ No newline at end of file diff --git a/docker-compose-tile-server.yml b/docker-compose-tile-server.yml new file mode 100644 index 0000000..421a3a2 --- /dev/null +++ b/docker-compose-tile-server.yml @@ -0,0 +1,14 @@ +# Tile Server +# +# Access via "http://localhost:8091" +version: '3' +services: + Tile-Server: + image: klokantech/openmaptiles-server + ports: + - 8091:80 + volumes: + - ./maps-data:/data + restart: always + cpus: 1.4 + cpuset: '1' diff --git a/docker-compose.yml b/docker-compose.yml index 960e08d..a259b52 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,14 +1,13 @@ # MediaWiki with MariaDB -# -# Access via "http://localhost:8080" -# (or "http://$(docker-machine ip):8080" if using docker-machine) + version: '3' + services: + mediawiki: build: . - restart: always ports: - - 8090:80 + - '8090:80' links: - database volumes: @@ -17,18 +16,18 @@ services: - ./wikidata/Wiki_Logo.png:/var/www/html/resources/assets/Wiki_Logo.png - ./wikidata/extensions:/var/www/html/extensions - ./wikidata/vendor:/var/www/html/vendor - # After initial setup, download LocalSettings.php to the same directory as - # this yaml and uncomment the following line and use compose to restart - # the mediawiki service +# After initial setup, download LocalSettings.php to the same directory as +# this yaml and uncomment the following line and use compose to restart +# the mediawiki service - ./wikidata/LocalSettings.php:/var/www/html/LocalSettings.php + restart: always database: image: mariadb - restart: always environment: - # @see https://phabricator.wikimedia.org/source/mediawiki/browse/master/includes/DefaultSettings.php MYSQL_DATABASE: rd13mediawikidb MYSQL_USER: wikiuser MYSQL_PASSWORD: mwdbpw MYSQL_RANDOM_ROOT_PASSWORD: 'yes' volumes: - ./mdbdata:/var/lib/mysql + restart: always diff --git a/maps-data/config.json b/maps-data/config.json new file mode 100644 index 0000000..9e0af91 --- /dev/null +++ b/maps-data/config.json @@ -0,0 +1,38 @@ +{ + "styles": { + "standard": [ + "dark-matter", + "klokantech-basic", + "osm-bright", + "positron" + ], + "custom": [], + "lang": "de", + "langLatin": true, + "langAlts": true + }, + "settings": { + "serve": { + "vector": false, + "raster": true, + "services": false, + "static": false + }, + "raster": { + "format": "PNG_256", + "hidpi": 2, + "maxsize": 2048 + }, + "server": { + "title": "", + "redirect": "", + "domains": [] + }, + "memcache": { + "size": 3, + "servers": [ + "localhost:11211" + ] + } + } +} \ No newline at end of file diff --git a/tile-server/apache.conf b/tile-server/apache.conf new file mode 100644 index 0000000..07fc520 --- /dev/null +++ b/tile-server/apache.conf @@ -0,0 +1,17 @@ + + ServerAdmin webmaster@localhost + + LoadTileConfigFile /usr/local/etc/renderd.conf + ModTileRenderdSocketName /var/run/renderd/renderd.sock + ModTileRequestTimeout 0 + ModTileMissingRequestTimeout 30 + + DocumentRoot /var/www/html + + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + + + Header set Access-Control-Allow-Origin "*" + + \ No newline at end of file diff --git a/tile-server/docker-compose-import-static.yml b/tile-server/docker-compose-import-static.yml new file mode 100644 index 0000000..1dc8006 --- /dev/null +++ b/tile-server/docker-compose-import-static.yml @@ -0,0 +1,15 @@ +# Tile Server +# +# Access via "http://localhost:8091" +version: '3' +services: + OSM-tile-Server: + build: . + ports: + - 8091:80 + volumes: + - ./OSM-maps-data:/var/lib/postgresql/12/main + command: "run" + restart: always + cpus: 1.4 + cpuset: '1' diff --git a/tile-server/docker-compose-import-with-download.yml b/tile-server/docker-compose-import-with-download.yml new file mode 100644 index 0000000..be7155a --- /dev/null +++ b/tile-server/docker-compose-import-with-download.yml @@ -0,0 +1,19 @@ +# Tile Server +# +# Access via "http://localhost:8091" +version: '3' +services: + OSM-tile-Server: + build: . + ports: + - 8091:80 + environment: + DOWNLOAD_PBF: "https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf" + DOWNLOAD_POLY: "https://download.geofabrik.de/europe/luxembourg.poly" + volumes: + - ./OSM-maps-data:/var/lib/postgresql/12/main + - ./OSM-maps-data/rendered:/var/lib/mod_tile + command: "import" + restart: always + cpus: 1.4 + cpuset: '1' \ No newline at end of file diff --git a/tile-server/docker-compose.yml b/tile-server/docker-compose.yml new file mode 100644 index 0000000..1dc8006 --- /dev/null +++ b/tile-server/docker-compose.yml @@ -0,0 +1,15 @@ +# Tile Server +# +# Access via "http://localhost:8091" +version: '3' +services: + OSM-tile-Server: + build: . + ports: + - 8091:80 + volumes: + - ./OSM-maps-data:/var/lib/postgresql/12/main + command: "run" + restart: always + cpus: 1.4 + cpuset: '1' diff --git a/tile-server/dockerfile b/tile-server/dockerfile new file mode 100644 index 0000000..e416f3a --- /dev/null +++ b/tile-server/dockerfile @@ -0,0 +1,184 @@ +FROM ubuntu:20.04 + +# Based on +# https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/ + +# Set up environment +ENV TZ=UTC +ENV AUTOVACUUM=on +ENV UPDATES=disabled +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Install dependencies +RUN apt-get update \ + && apt-get install -y wget gnupg2 lsb-core apt-transport-https ca-certificates curl \ + && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ + && echo "deb [ trusted=yes ] https://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list \ + && wget --quiet -O - https://deb.nodesource.com/setup_14.x | bash - \ + && apt-get update \ + && apt-get install -y nodejs + +RUN apt-get install -y --no-install-recommends \ + apache2 \ + apache2-dev \ + autoconf \ + build-essential \ + bzip2 \ + cmake \ + cron \ + fonts-noto-cjk \ + fonts-noto-hinted \ + fonts-noto-unhinted \ + gcc \ + gdal-bin \ + git-core \ + libagg-dev \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbz2-dev \ + libcairo-dev \ + libcairomm-1.0-dev \ + libexpat1-dev \ + libfreetype6-dev \ + libgdal-dev \ + libgeos++-dev \ + libgeos-dev \ + libicu-dev \ + liblua5.3-dev \ + libmapnik-dev \ + libpq-dev \ + libproj-dev \ + libprotobuf-c-dev \ + libtiff5-dev \ + libtool \ + libxml2-dev \ + lua5.3 \ + make \ + mapnik-utils \ + osmium-tool \ + osmosis \ + postgis \ + postgresql-12 \ + postgresql-contrib-12 \ + postgresql-server-dev-12 \ + protobuf-c-compiler \ + python3-mapnik \ + python3-lxml \ + python3-psycopg2 \ + python3-shapely \ + python3-pip \ + sudo \ + tar \ + ttf-unifont \ + unzip \ + wget \ + zlib1g-dev \ +&& apt-get clean autoclean \ +&& apt-get autoremove --yes \ +&& rm -rf /var/lib/{apt,dpkg,cache,log}/ + +# Install python libraries +RUN pip3 install requests \ + && pip3 install pyyaml + +# Set up PostGIS +RUN wget https://download.osgeo.org/postgis/source/postgis-3.1.1.tar.gz -O postgis.tar.gz \ + && mkdir -p postgis_src \ + && tar -xvzf postgis.tar.gz --strip 1 -C postgis_src \ + && rm postgis.tar.gz \ + && cd postgis_src \ + && ./configure \ + && make -j $(nproc) \ + && make -j $(nproc) install \ + && cd .. && rm -rf postgis_src + +# Set up renderer user +RUN adduser --disabled-password --gecos "" renderer + +# Install latest osm2pgsql +RUN mkdir -p /home/renderer/src \ + && cd /home/renderer/src \ + && git clone -b master https://github.com/openstreetmap/osm2pgsql.git --depth 1 \ + && cd /home/renderer/src/osm2pgsql \ + && rm -rf .git \ + && mkdir build \ + && cd build \ + && cmake .. \ + && make -j $(nproc) \ + && make -j $(nproc) install \ + && mkdir /nodes \ + && chown renderer:renderer /nodes \ + && rm -rf /home/renderer/src/osm2pgsql + +# Install mod_tile and renderd +RUN mkdir -p /home/renderer/src \ + && cd /home/renderer/src \ + && git clone -b switch2osm https://github.com/SomeoneElseOSM/mod_tile.git --depth 1 \ + && cd mod_tile \ + && rm -rf .git \ + && ./autogen.sh \ + && ./configure \ + && make -j $(nproc) \ + && make -j $(nproc) install \ + && make -j $(nproc) install-mod_tile \ + && ldconfig \ + && cd .. + +# Configure stylesheet +RUN mkdir -p /home/renderer/src \ + && cd /home/renderer/src \ + && git clone --single-branch --branch v5.3.1 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \ + && cd openstreetmap-carto \ + && rm -rf .git \ + && npm install -g carto@0.18.2 \ + && carto project.mml > mapnik.xml + +# Configure renderd +RUN sed -i 's/renderaccount/renderer/g' /usr/local/etc/renderd.conf \ + && sed -i 's/\/truetype//g' /usr/local/etc/renderd.conf \ + && sed -i 's/hot/tile/g' /usr/local/etc/renderd.conf + +# Configure Apache +RUN mkdir /var/lib/mod_tile \ + && chown renderer /var/lib/mod_tile \ + && mkdir /var/run/renderd \ + && chown renderer /var/run/renderd \ + && echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/apache2/conf-available/mod_tile.conf \ + && echo "LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so" >> /etc/apache2/conf-available/mod_headers.conf \ + && a2enconf mod_tile && a2enconf mod_headers +COPY apache.conf /etc/apache2/sites-available/000-default.conf +COPY leaflet-demo.html /var/www/html/index.html +RUN ln -sf /dev/stdout /var/log/apache2/access.log \ + && ln -sf /dev/stderr /var/log/apache2/error.log + +# Configure PosgtreSQL +COPY postgresql.custom.conf.tmpl /etc/postgresql/12/main/ +RUN chown -R postgres:postgres /var/lib/postgresql \ + && chown postgres:postgres /etc/postgresql/12/main/postgresql.custom.conf.tmpl \ + && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/12/main/pg_hba.conf \ + && echo "host all all ::/0 md5" >> /etc/postgresql/12/main/pg_hba.conf + +# Copy update scripts +COPY openstreetmap-tiles-update-expire /usr/bin/ +RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire \ + && mkdir /var/log/tiles \ + && chmod a+rw /var/log/tiles \ + && ln -s /home/renderer/src/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag \ + && echo "* * * * * renderer openstreetmap-tiles-update-expire\n" >> /etc/crontab + +# Install trim_osc.py helper script +RUN mkdir -p /home/renderer/src \ + && cd /home/renderer/src \ + && git clone https://github.com/zverik/regional \ + && cd regional \ + && git checkout 889d630a1e1a1bacabdd1dad6e17b49e7d58cd4b \ + && rm -rf .git \ + && chmod u+x /home/renderer/src/regional/trim_osc.py + +# Start running +COPY run.sh / +COPY indexes.sql / +ENTRYPOINT ["/run.sh"] +CMD [] + +EXPOSE 80 5432 \ No newline at end of file diff --git a/tile-server/indexes.sql b/tile-server/indexes.sql new file mode 100644 index 0000000..d387207 --- /dev/null +++ b/tile-server/indexes.sql @@ -0,0 +1,48 @@ +-- Extracted from https://github.com/gravitystorm/openstreetmap-carto +-- Comes with a CC0 license + +-- These are optional but suggested indexes for rendering OpenStreetMap Carto +-- with a full planet database. +-- This file is generated with scripts/indexes.py + +CREATE INDEX planet_osm_roads_admin + ON planet_osm_roads USING GIST (way) + WHERE boundary = 'administrative'; +CREATE INDEX planet_osm_roads_roads_ref + ON planet_osm_roads USING GIST (way) + WHERE highway IS NOT NULL AND ref IS NOT NULL; +CREATE INDEX planet_osm_roads_admin_low + ON planet_osm_roads USING GIST (way) + WHERE boundary = 'administrative' AND admin_level IN ('0', '1', '2', '3', '4'); +CREATE INDEX planet_osm_line_ferry + ON planet_osm_line USING GIST (way) + WHERE route = 'ferry'; +CREATE INDEX planet_osm_line_river + ON planet_osm_line USING GIST (way) + WHERE waterway = 'river'; +CREATE INDEX planet_osm_line_name + ON planet_osm_line USING GIST (way) + WHERE name IS NOT NULL; +CREATE INDEX planet_osm_polygon_water + ON planet_osm_polygon USING GIST (way) + WHERE waterway IN ('dock', 'riverbank', 'canal') + OR landuse IN ('reservoir', 'basin') + OR "natural" IN ('water', 'glacier'); +CREATE INDEX planet_osm_polygon_nobuilding + ON planet_osm_polygon USING GIST (way) + WHERE building IS NULL; +CREATE INDEX planet_osm_polygon_name + ON planet_osm_polygon USING GIST (way) + WHERE name IS NOT NULL; +CREATE INDEX planet_osm_polygon_way_area_z10 + ON planet_osm_polygon USING GIST (way) + WHERE way_area > 23300; +CREATE INDEX planet_osm_polygon_military + ON planet_osm_polygon USING GIST (way) + WHERE (landuse = 'military' OR military = 'danger_area') AND building IS NULL; +CREATE INDEX planet_osm_polygon_way_area_z6 + ON planet_osm_polygon USING GIST (way) + WHERE way_area > 5980000; +CREATE INDEX planet_osm_point_place + ON planet_osm_point USING GIST (way) + WHERE place IS NOT NULL AND name IS NOT NULL; \ No newline at end of file diff --git a/tile-server/leaflet-demo.html b/tile-server/leaflet-demo.html new file mode 100644 index 0000000..60d47ec --- /dev/null +++ b/tile-server/leaflet-demo.html @@ -0,0 +1,35 @@ + + + + Custom Tile Server + + + + + + + + + + + +
+ + + + \ No newline at end of file diff --git a/tile-server/openstreetmap-tiles-update-expire b/tile-server/openstreetmap-tiles-update-expire new file mode 100644 index 0000000..9ae5ca8 --- /dev/null +++ b/tile-server/openstreetmap-tiles-update-expire @@ -0,0 +1,206 @@ +#!/bin/sh + +set -e + +#------------------------------------------------------------------------------ +# AJT - change directory to mod_tile directory so that we can run replag +# and other things directly from this script when run from cron. +# Change the actual location to wherever installed locally. +#------------------------------------------------------------------------------ +export PATH=.:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ACCOUNT=renderer +cd /home/$ACCOUNT/src/mod_tile/ + +#------------------------------------------------------------------------------ +# Extra OSM2PGSQL_OPTIONS may need setting if a tag transform script is +# in use. See https://github.com/SomeoneElseOSM/SomeoneElse-style and +# http://wiki.openstreetmap.org/wiki/User:SomeoneElse/Ubuntu_1404_tileserver_load +# The database name always needs setting. +#------------------------------------------------------------------------------ +OSMOSIS_BIN=osmosis +OSM2PGSQL_BIN=osm2pgsql +TRIM_BIN=/home/$ACCOUNT/src/regional/trim_osc.py + +DBNAME=gis +OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style ${OSM2PGSQL_EXTRA_ARGS}" + +#------------------------------------------------------------------------------ +# When using trim_osc.py we can define either a bounding box (such as this +# example for England and Wales) or a polygon. +# See https://github.com/zverik/regional . +# This area will usually correspond to the data originally loaded. +#------------------------------------------------------------------------------ +TRIM_POLY_FILE="/var/lib/mod_tile/data.poly" +TRIM_OPTIONS="-d $DBNAME" +#TRIM_REGION_OPTIONS="-b -14.17 48.85 2.12 61.27" +TRIM_REGION_OPTIONS="-p $TRIM_POLY_FILE" + +BASE_DIR=/var/lib/mod_tile +LOG_DIR=/var/log/tiles/ +WORKOSM_DIR=$BASE_DIR/.osmosis + +LOCK_FILE=/tmp/openstreetmap-update-expire-lock.txt +CHANGE_FILE=$BASE_DIR/changes.osc.gz +EXPIRY_FILE=$BASE_DIR/dirty_tiles +STOP_FILE=$BASE_DIR/stop.txt + +OSMOSISLOG=$LOG_DIR/osmosis.log +PGSQLLOG=$LOG_DIR/osm2pgsql.log +EXPIRYLOG=$LOG_DIR/expiry.log +RUNLOG=$LOG_DIR/run.log + +#------------------------------------------------------------------------------ +# The tile expiry section below can re-render, delete or dirty expired tiles. +# By default, tiles between EXPIRY_MINZOOM and EXPIRY_MAXZOOM are rerendered. +# "render_expired" can optionally delete (and/or dirty) tiles above a certail +# threshold rather than rendering them. +# Here we expire (but don't immediately rerender) tiles between zoom levels +# 13 and 18 and delete between 19 and 20. +#------------------------------------------------------------------------------ +EXPIRY_MINZOOM=13 +EXPIRY_TOUCHFROM=13 +EXPIRY_DELETEFROM=19 +EXPIRY_MAXZOOM=20 + +#************************************************************************* +#************************************************************************* + +m_info() +{ + echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG" +} + +m_error() +{ + echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ [error] $1" >> "$RUNLOG" + + m_info "resetting state" + /bin/cp $WORKOSM_DIR/last.state.txt $WORKOSM_DIR/state.txt || true + + rm "$CHANGE_FILE" || true + rm "$EXPIRY_FILE.$$" || true + rm "$LOCK_FILE" + exit +} + +m_ok() +{ + echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG" +} + +getlock() +{ + if [ -s $1 ]; then + if [ "$(ps -p `cat $1` | wc -l)" -gt 1 ]; then + return 1 #false + fi + fi + + echo $$ >"$1" + return 0 #true +} + +freelock() +{ + rm "$1" + rm "$CHANGE_FILE" +} + + +if [ $# -eq 1 ] ; then + m_info "Initialising Osmosis replication system to $1" + mkdir $WORKOSM_DIR + $OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG" + wget "https://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt + mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt + sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt +else +# make sure the lockfile is removed when we exit and then claim it + + if ! getlock "$LOCK_FILE"; then + m_info "pid `cat $LOCK_FILE` still running" + exit 3 + fi + + if [ -e $STOP_FILE ]; then + m_info "stopped" + exit 2 + fi + +# ----------------------------------------------------------------------------- +# Add disk space check from https://github.com/zverik/regional +# ----------------------------------------------------------------------------- +MIN_DISK_SPACE_MB=500 + +if `python -c "import os, sys; st=os.statvfs('$BASE_DIR'); sys.exit(1 if st.f_bavail*st.f_frsize/1024/1024 > $MIN_DISK_SPACE_MB else 0)"`; then + m_info "there is less than $MIN_DISK_SPACE_MB MB left" + exit 4 +fi + + seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2` + + m_ok "start import from seq-nr $seq, replag is `osmosis-db_replag -h`" + + /bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt + m_ok "downloading diff" + + if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then + m_error "Osmosis error" + fi + +if [ -f $TRIM_POLY_FILE ] ; then + m_ok "filtering diff" + if ! $TRIM_BIN $TRIM_OPTIONS $TRIM_REGION_OPTIONS -z $CHANGE_FILE $CHANGE_FILE 1>&2 2>> "$RUNLOG"; then + m_error "Trim_osc error" + fi +else + m_ok "filtering diff skipped" +fi + m_ok "importing diff" +#------------------------------------------------------------------------------ +# Previously openstreetmap-tiles-update-expire tried to dirty layer +# "$EXPIRY_MAXZOOM - 3" (which was 15) only. Instead we write all expired +# tiles in range to the list (note the "-" rather than ":" in the "-e" +# parameter). +#------------------------------------------------------------------------------ + if ! $OSM2PGSQL_BIN -a --slim -e$EXPIRY_MINZOOM-$EXPIRY_MAXZOOM $OSM2PGSQL_OPTIONS -o "$EXPIRY_FILE.$$" $CHANGE_FILE 1>&2 2> "$PGSQLLOG"; then + m_error "osm2pgsql error" + fi + +#------------------------------------------------------------------------------ +# The lockfile is normally removed before we expire tiles because that is +# something thatcan be done in parallel with further processing. In order to +# avoid rework, if actually rerendering is done rather than just deleting or +# dirtying, it makes sense to move it lower down. +#------------------------------------------------------------------------------ +# m_ok "Import complete; removing lock file" +# freelock "$LOCK_FILE" + + m_ok "expiring tiles" +#------------------------------------------------------------------------------ +# When expiring tiles we need to define the style sheet if it's not "default". +# In this case it's "ajt". +# Previously all tiles on the "dirty" list between $EXPIRY_MINZOOM and +# $EXPIRY_MAXZOOM were dirtied. We currently re-render +# tiles >= $EXPIRY_MINZOOM and < $EXPIRY_DELETEFROM, expiry from 14 and +# delete >= $EXPIRY_DELETEFROM and <= $EXPIRY_MAXZOOM. +# The default path to renderd.sock is fixed. +#------------------------------------------------------------------------------ + if ! render_expired --map=ajt --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then + m_info "Expiry failed" + fi + + rm "$EXPIRY_FILE.$$" + +#------------------------------------------------------------------------------ +# Only remove the lock file after expiry (if system is slow we want to delay +# the next import, not have multiple render_expired processes running) +#------------------------------------------------------------------------------ + freelock "$LOCK_FILE" + + m_ok "Done with import" + + + + +fi \ No newline at end of file diff --git a/tile-server/postgresql.custom.conf.tmpl b/tile-server/postgresql.custom.conf.tmpl new file mode 100644 index 0000000..df9c9b6 --- /dev/null +++ b/tile-server/postgresql.custom.conf.tmpl @@ -0,0 +1,25 @@ +# Suggested minimal settings from +# https://ircama.github.io/osm-carto-tutorials/tile-server-ubuntu/ + +shared_buffers = 128MB +min_wal_size = 1GB +max_wal_size = 2GB +maintenance_work_mem = 256MB + +# Suggested settings from +# https://github.com/openstreetmap/chef/blob/master/roles/tile.rb#L38-L45 + +max_connections = 250 +temp_buffers = 32MB +work_mem = 128MB +wal_buffers = 1024kB +wal_writer_delay = 500ms +commit_delay = 10000 +# checkpoint_segments = 60 # unrecognized in psql 10.7.1 +max_wal_size = 2880MB +random_page_cost = 1.1 +track_activity_query_size = 16384 +autovacuum_vacuum_scale_factor = 0.05 +autovacuum_analyze_scale_factor = 0.02 + +listen_addresses = '*' \ No newline at end of file diff --git a/tile-server/run.sh b/tile-server/run.sh new file mode 100644 index 0000000..85c4971 --- /dev/null +++ b/tile-server/run.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +set -x + +function createPostgresConfig() { + cp /etc/postgresql/12/main/postgresql.custom.conf.tmpl /etc/postgresql/12/main/conf.d/postgresql.custom.conf + sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/12/main/conf.d/postgresql.custom.conf + cat /etc/postgresql/12/main/conf.d/postgresql.custom.conf +} + +function setPostgresPassword() { + sudo -u postgres psql -c "ALTER USER renderer PASSWORD '${PGPASSWORD:-renderer}'" +} + +if [ "$#" -ne 1 ]; then + echo "usage: " + echo "commands:" + echo " import: Set up the database and import /data.osm.pbf" + echo " run: Runs Apache and renderd to serve tiles at /tile/{z}/{x}/{y}.png" + echo "environment variables:" + echo " THREADS: defines number of threads used for importing / tile rendering" + echo " UPDATES: consecutive updates (enabled/disabled)" + exit 1 +fi + +if [ "$1" = "import" ]; then + # Ensure that database directory is in right state + chown postgres:postgres -R /var/lib/postgresql + if [ ! -f /var/lib/postgresql/12/main/PG_VERSION ]; then + sudo -u postgres /usr/lib/postgresql/12/bin/pg_ctl -D /var/lib/postgresql/12/main/ initdb -o "--locale C.UTF-8" + fi + + # Initialize PostgreSQL + createPostgresConfig + service postgresql start + sudo -u postgres createuser renderer + sudo -u postgres createdb -E UTF8 -O renderer gis + sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis;" + sudo -u postgres psql -d gis -c "CREATE EXTENSION hstore;" + sudo -u postgres psql -d gis -c "ALTER TABLE geometry_columns OWNER TO renderer;" + sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO renderer;" + setPostgresPassword + + # Download Luxembourg as sample if no data is provided + if [ ! -f /data.osm.pbf ] && [ -z "$DOWNLOAD_PBF" ]; then + echo "WARNING: No import file at /data.osm.pbf, so importing Luxembourg as example..." + DOWNLOAD_PBF="https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf" + DOWNLOAD_POLY="https://download.geofabrik.de/europe/luxembourg.poly" + fi + + if [ -n "$DOWNLOAD_PBF" ]; then + echo "INFO: Download PBF file: $DOWNLOAD_PBF" + wget "$WGET_ARGS" "$DOWNLOAD_PBF" -O /data.osm.pbf + if [ -n "$DOWNLOAD_POLY" ]; then + echo "INFO: Download PBF-POLY file: $DOWNLOAD_POLY" + wget "$WGET_ARGS" "$DOWNLOAD_POLY" -O /data.poly + fi + fi + + if [ "$UPDATES" = "enabled" ]; then + # determine and set osmosis_replication_timestamp (for consecutive updates) + osmium fileinfo /data.osm.pbf > /var/lib/mod_tile/data.osm.pbf.info + osmium fileinfo /data.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44 > /var/lib/mod_tile/replication_timestamp.txt + REPLICATION_TIMESTAMP=$(cat /var/lib/mod_tile/replication_timestamp.txt) + + # initial setup of osmosis workspace (for consecutive updates) + sudo -u renderer openstreetmap-tiles-update-expire $REPLICATION_TIMESTAMP + fi + + # copy polygon file if available + if [ -f /data.poly ]; then + sudo -u renderer cp /data.poly /var/lib/mod_tile/data.poly + fi + + # Import data + sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style /data.osm.pbf ${OSM2PGSQL_EXTRA_ARGS} + + # Create indexes + sudo -u postgres psql -d gis -f indexes.sql + + #Import external data + sudo chown -R renderer: /home/renderer/src + sudo -u renderer python3 /home/renderer/src/openstreetmap-carto/scripts/get-external-data.py -c /home/renderer/src/openstreetmap-carto/external-data.yml -D /home/renderer/src/openstreetmap-carto/data + + # Register that data has changed for mod_tile caching purposes + touch /var/lib/mod_tile/planet-import-complete + + service postgresql stop + + exit 0 +fi + +if [ "$1" = "run" ]; then + # Clean /tmp + rm -rf /tmp/* + + # Fix postgres data privileges + chown postgres:postgres /var/lib/postgresql -R + + # Configure Apache CORS + if [ "$ALLOW_CORS" == "enabled" ] || [ "$ALLOW_CORS" == "1" ]; then + echo "export APACHE_ARGUMENTS='-D ALLOW_CORS'" >> /etc/apache2/envvars + fi + + # Initialize PostgreSQL and Apache + createPostgresConfig + service postgresql start + service apache2 restart + setPostgresPassword + + # Configure renderd threads + sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /usr/local/etc/renderd.conf + + # start cron job to trigger consecutive updates + if [ "$UPDATES" = "enabled" ] || [ "$UPDATES" = "1" ]; then + /etc/init.d/cron start + fi + + # Run while handling docker stop's SIGTERM + stop_handler() { + kill -TERM "$child" + } + trap stop_handler SIGTERM + + sudo -u renderer renderd -f -c /usr/local/etc/renderd.conf & + child=$! + wait "$child" + + service postgresql stop + + exit 0 +fi + +echo "invalid command" +exit 1 \ No newline at end of file