Conrad Schulz
e022f839db
BREAKING CHANGE: satellite.mbtiles ist nun einheitlich JPEG (zuvor gemischte PNG/JPEG) Features: - Neue Scripts: convert-satellite-to-jpeg.py (batch-basiert, resume-fähig) - download-satellite.py jetzt mit on-the-fly PNG→JPEG-Konvertierung - docker-compose.download.yml: Pillow installiert, Qualität konfigurierbar Fixes: - 1.505.049 PNG-Tiles zu JPEG konvertiert (22.9 Minuten) - SQLite quick_check bestanden (png=0, quick_check=ok, total=5.6M) - Martin lädt satellite-Source ohne Format-Warnungen - Alle 3 Sources (osm, osm-europe, satellite) geladen und produktiv Documentation: - ADMIN.md: Satellit-Konvertierungs-Anleitung aktualisiert - PROJECT_CONTEXT.md: Finaler Status dokumentiert - Session-Log: 2026-06-15_satellite-fix-cleanup_session.md Cleanup: - Temporäre Dateien gelöscht - 17GB Backup gelöscht - __pycache__ gelöscht
113 lines
No EOL
3.5 KiB
Python
113 lines
No EOL
3.5 KiB
Python
#!/usr/bin/env python3
|
|
import io
|
|
import os
|
|
import signal
|
|
import sqlite3
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from PIL import Image
|
|
|
|
DB = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/data/satellite.mbtiles")
|
|
BACKUP = Path(os.environ.get("SAT_BACKUP", str(DB) + ".before-png-to-jpg-fix"))
|
|
BATCH = int(os.environ.get("SAT_CONVERT_BATCH", "500"))
|
|
JPEG_QUALITY = int(os.environ.get("SAT_JPEG_QUALITY", "90"))
|
|
SLEEP_SECONDS = float(os.environ.get("SAT_CONVERT_SLEEP", "0"))
|
|
REQUIRE_BACKUP = os.environ.get("SAT_REQUIRE_BACKUP", "1") == "1"
|
|
|
|
PNG_SIG = b"\x89PNG\r\n\x1a\n"
|
|
STOP = False
|
|
|
|
|
|
def log(message):
|
|
print(message, flush=True)
|
|
|
|
|
|
def handle_stop(signum, frame):
|
|
global STOP
|
|
STOP = True
|
|
log("[Convert] Stop requested; finishing current batch before exit")
|
|
|
|
|
|
def png_to_jpeg(data):
|
|
with Image.open(io.BytesIO(data)) as img:
|
|
if img.mode == "RGBA" or "transparency" in img.info:
|
|
rgba = img.convert("RGBA")
|
|
background = Image.new("RGBA", rgba.size, (0, 0, 0, 255))
|
|
background.alpha_composite(rgba)
|
|
rgb = background.convert("RGB")
|
|
else:
|
|
rgb = img.convert("RGB")
|
|
|
|
out = io.BytesIO()
|
|
rgb.save(out, format="JPEG", quality=JPEG_QUALITY, optimize=False)
|
|
return out.getvalue()
|
|
|
|
|
|
def main():
|
|
if REQUIRE_BACKUP and not BACKUP.exists():
|
|
raise SystemExit("[Convert] Backup missing: %s" % BACKUP)
|
|
|
|
signal.signal(signal.SIGTERM, handle_stop)
|
|
signal.signal(signal.SIGINT, handle_stop)
|
|
|
|
started = time.time()
|
|
converted = 0
|
|
scanned_png = 0
|
|
batch = []
|
|
|
|
conn = sqlite3.connect(str(DB))
|
|
conn.execute("PRAGMA busy_timeout=60000")
|
|
journal_mode = conn.execute("PRAGMA journal_mode=DELETE").fetchone()[0]
|
|
conn.execute("PRAGMA synchronous=NORMAL")
|
|
|
|
before = conn.execute("SELECT value FROM metadata WHERE name='format'").fetchone()[0]
|
|
conn.execute("UPDATE metadata SET value='jpg' WHERE name='format'")
|
|
conn.commit()
|
|
log("[Convert] DB=%s" % DB)
|
|
log("[Convert] backup=%s" % BACKUP)
|
|
log("[Convert] batch=%d quality=%d sleep=%.3fs journal=%s" % (
|
|
BATCH, JPEG_QUALITY, SLEEP_SECONDS, journal_mode))
|
|
log("[Convert] metadata.format %s -> jpg" % before)
|
|
|
|
try:
|
|
for rowid, data in conn.execute("SELECT rowid, tile_data FROM tiles"):
|
|
if STOP:
|
|
break
|
|
if not data.startswith(PNG_SIG):
|
|
continue
|
|
|
|
scanned_png += 1
|
|
batch.append((png_to_jpeg(data), rowid))
|
|
|
|
if len(batch) >= BATCH:
|
|
conn.executemany("UPDATE tiles SET tile_data=? WHERE rowid=?", batch)
|
|
conn.commit()
|
|
converted += len(batch)
|
|
batch.clear()
|
|
elapsed = time.time() - started
|
|
rate = converted / elapsed if elapsed else 0
|
|
log("[Convert] converted=%d rate=%.1f/s elapsed=%.1fm" % (
|
|
converted, rate, elapsed / 60))
|
|
if SLEEP_SECONDS:
|
|
time.sleep(SLEEP_SECONDS)
|
|
|
|
if batch:
|
|
conn.executemany("UPDATE tiles SET tile_data=? WHERE rowid=?", batch)
|
|
conn.commit()
|
|
converted += len(batch)
|
|
|
|
conn.execute("ANALYZE")
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
elapsed = time.time() - started
|
|
status = "stopped" if STOP else "done"
|
|
log("[Convert] %s scanned_png=%d converted=%d elapsed=%.1fm" % (
|
|
status, scanned_png, converted, elapsed / 60))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |