synq-core-os/scripts/prefetch_satellite_regions.py

#!/usr/bin/env python3
"""Pre-fetch satellite (MODIS/VIIRS TrueColor) tiles for key regions.

Downloads directly from NASA GIBS with concurrent workers and local caching.
Regions: Southern California, GCC, Iran, Israel
"""

import math
import requests
from pathlib import Path
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from threading import Lock

CACHE_DIR = Path("/home/raider1984/synq-data/pmtiles/tiles/satellite")
TILE_URL = "https://gibs.earthdata.nasa.gov/wmts/epsg3857/best/VIIRS_SNPP_CorrectedReflectance_TrueColor/default/GoogleMapsCompatible_Level9/{z}/{x}/{y}.jpeg"
MAX_ZOOM = 9

REGIONS = {
    "southern_california": (32.0, 35.5, -121.0, -114.0),
    "gcc": (16.0, 32.5, 47.0, 57.0),
    "iran": (25.0, 40.0, 44.0, 63.5),
    "israel": (29.5, 34.0, 34.0, 36.5),
}

ZOOM_LEVELS = [5, 6, 7, 8, 9]
MAX_WORKERS = 16
SESSION = requests.Session()
SESSION.headers.update({"User-Agent": "Synq-Intel-TilePrefetch/1.0"})

stats_lock = Lock()


def latlng_to_tile(lat, lng, zoom):
    n = 2 ** zoom
    x = int((lng + 180.0) / 360.0 * n)
    lat_rad = math.radians(lat)
    y = int((1.0 - math.log(math.tan(lat_rad) + 1.0 / math.cos(lat_rad)) / math.pi) / 2.0 * n)
    return x, y


def get_tile_range(lat_min, lat_max, lng_min, lng_max, zoom):
    x1, y1 = latlng_to_tile(lat_max, lng_min, zoom)
    x2, y2 = latlng_to_tile(lat_min, lng_max, zoom)
    return min(x1, x2), max(x1, x2), min(y1, y2), max(y1, y2)


def fetch_tile(z, x, y):
    cache_path = CACHE_DIR / str(z) / str(x) / f"{y}.jpg"
    if cache_path.exists():
        return True, "cached", z

    url = TILE_URL.format(z=z, x=x, y=y)
    try:
        r = SESSION.get(url, timeout=20)
        if r.status_code == 200:
            cache_path.parent.mkdir(parents=True, exist_ok=True)
            with open(cache_path, "wb") as f:
                f.write(r.content)
            return True, "downloaded", z
        else:
            return False, f"http_{r.status_code}", z
    except Exception as e:
        return False, str(e)[:50], z


def prefetch_region(name, lat_min, lat_max, lng_min, lng_max, zoom_levels):
    print(f"\n📍 {name.upper()}")
    print(f"   Bounds: lat {lat_min}–{lat_max}, lng {lng_min}–{lng_max}")
    sys.stdout.flush()

    all_tiles = []
    for z in zoom_levels:
        if z > MAX_ZOOM:
            continue
        x_min, x_max, y_min, y_max = get_tile_range(lat_min, lat_max, lng_min, lng_max, z)
        max_tile = (2 ** z) - 1
        x_min = max(0, x_min)
        x_max = min(max_tile, x_max)
        y_min = max(0, y_min)
        y_max = min(max_tile, y_max)
        for x in range(x_min, x_max + 1):
            for y in range(y_min, y_max + 1):
                all_tiles.append((z, x, y))

    zoom_stats = {z: {"total": 0, "downloaded": 0, "cached": 0, "errors": 0} for z in zoom_levels if z <= MAX_ZOOM}
    total_tiles = len(all_tiles)
    completed = 0
    downloaded = 0
    cached = 0
    errors = 0

    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        future_to_tile = {executor.submit(fetch_tile, z, x, y): (z, x, y) for z, x, y in all_tiles}
        for future in as_completed(future_to_tile):
            success, status, z = future.result()
            with stats_lock:
                completed += 1
                zoom_stats[z]["total"] += 1
                if success:
                    if status == "downloaded":
                        downloaded += 1
                        zoom_stats[z]["downloaded"] += 1
                    else:
                        cached += 1
                        zoom_stats[z]["cached"] += 1
                else:
                    errors += 1
                    zoom_stats[z]["errors"] += 1

                if completed % 100 == 0 or completed == total_tiles:
                    pct = completed * 100 // total_tiles if total_tiles > 0 else 0
                    print(f"\r   Progress: {completed}/{total_tiles} ({pct}%) | DL:{downloaded} | Cache:{cached} | Err:{errors}", end="")
                    sys.stdout.flush()

    print()
    for z in zoom_levels:
        if z > MAX_ZOOM:
            continue
        st = zoom_stats[z]
        if st["total"] > 0:
            print(f"   Zoom {z:2d}: {st['total']:5d} tiles | {st['downloaded']:5d} downloaded | {st['cached']:5d} cached | {st['errors']:3d} errors")

    print(f"   TOTAL: {total_tiles} tiles | {downloaded} downloaded | {cached} cached | {errors} errors")
    return total_tiles, downloaded, cached, errors


if __name__ == "__main__":
    print("=" * 70)
    print("NASA GIBS SATELLITE TILE PREFETCH (VIIRS SNPP TrueColor)")
    print(f"Workers: {MAX_WORKERS} | Zooms: {ZOOM_LEVELS} | MaxZoom: {MAX_ZOOM}")
    print("=" * 70)
    sys.stdout.flush()

    grand_total = [0, 0, 0, 0]

    for name, (lat_min, lat_max, lng_min, lng_max) in REGIONS.items():
        counts = prefetch_region(name, lat_min, lat_max, lng_min, lng_max, ZOOM_LEVELS)
        for i in range(4):
            grand_total[i] += counts[i]

    print("\n" + "=" * 70)
    print("GRAND TOTAL")
    print(f"  {grand_total[0]} tiles checked")
    print(f"  {grand_total[1]} downloaded")
    print(f"  {grand_total[2]} already cached")
    print(f"  {grand_total[3]} errors")
    print(f"\nCache location: {CACHE_DIR}")
    print("=" * 70)