commit 9756538f16ae5a1aae7d35a32183404125730e63 Author: dev Date: Mon Mar 16 17:04:16 2026 +0000 Initial commit: Venus OS boat addons monorepo Organizes 11 projects for Cerbo GX/Venus OS into a single repository: - axiom-nmea: Raymarine LightHouse protocol decoder - dbus-generator-ramp: Generator current ramp controller - dbus-lightning: Blitzortung lightning monitor - dbus-meteoblue-forecast: Meteoblue weather forecast - dbus-no-foreign-land: noforeignland.com tracking - dbus-tides: Tide prediction from depth + harmonics - dbus-vrm-history: VRM cloud history proxy - dbus-windy-station: Windy.com weather upload - mfd-custom-app: MFD app deployment package - venus-html5-app: Custom Victron HTML5 app fork - watermaker: Watermaker PLC control UI Adds root README, .gitignore, project template, and per-project .gitignore files. Sensitive config files excluded via .gitignore with .example templates provided. Made-with: Cursor diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b11d1ac --- /dev/null +++ b/.gitignore @@ -0,0 +1,53 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg +*.egg-info/ +venv/ +.venv/ + +# Node +node_modules/ +dist/ +build/ +*.tgz +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Venus OS runtime (created by install.sh on device) +ext/velib_python + +# Build artifacts +*.tar.gz +*.sha256 + +# IDE +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Environment / secrets +.env.local +.env.development.local +.env.test.local +.env.production.local + +# Sensitive config files (use *.example.json templates instead) +dbus-meteoblue-forecast/forecast_config.json +dbus-windy-station/station_config.json + +# Databases +*.db + +# Design reference assets (kept locally, not in repo) +inspiration assets/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..1b9fd67 --- /dev/null +++ b/README.md @@ -0,0 +1,79 @@ +# Venus OS Boat Addons + +Custom addons for a Cerbo GX running [Venus OS](https://github.com/victronenergy/venus) on a sailboat. These services extend the system with weather monitoring, tide prediction, navigation tracking, generator management, and custom MFD displays. + +All D-Bus services follow the same deployment pattern: build a `.tar.gz` package, copy it to `/data/` on the Cerbo GX, and run `install.sh`. Services are managed by daemontools and survive firmware updates via `rc.local`. + +## Projects + +| Project | Type | Language | Description | +|---------|------|----------|-------------| +| [axiom-nmea](axiom-nmea/) | Library + Service | Python | Decodes Raymarine LightHouse protobuf multicast into NMEA 0183 sentences and Venus OS D-Bus services. Includes protocol documentation, debug tools, and a deployable D-Bus publisher. | +| [dbus-generator-ramp](dbus-generator-ramp/) | D-Bus Service | Python | Gradually ramps inverter/charger input current when running on generator to avoid overload. Features warm-up hold, overload detection with fast recovery, and a persistent power correlation learning model. | +| [dbus-lightning](dbus-lightning/) | D-Bus Service | Python | Monitors real-time lightning strikes from the Blitzortung network via WebSocket. Filters by distance, analyzes storm approach speed, and estimates ETA. | +| [dbus-meteoblue-forecast](dbus-meteoblue-forecast/) | D-Bus Service | Python | Fetches 7-day weather forecasts from the Meteoblue API (wind, waves, precipitation, temperature). Adjusts refresh rate based on boat movement. | +| [dbus-no-foreign-land](dbus-no-foreign-land/) | D-Bus Service | Python | Sends GPS position and track data to noforeignland.com. Includes a QML settings page for the Venus OS GUI. | +| [dbus-tides](dbus-tides/) | D-Bus Service | Python | Predicts tides by combining depth sensor readings with harmonic tidal models (NOAA stations and coastal grid). Records depth history in SQLite, detects high/low tides, and calibrates to chart depth. | +| [dbus-vrm-history](dbus-vrm-history/) | D-Bus Service | Python | Proxies historical data from the VRM cloud API and exposes it on D-Bus/MQTT for the frontend dashboard. | +| [dbus-windy-station](dbus-windy-station/) | D-Bus Service | Python | Uploads weather observations from Raymarine sensors to Windy.com Stations API. Supports both legacy and v2 Venus OS GUI plugins. | +| [mfd-custom-app](mfd-custom-app/) | Deployment Package | Shell | Builds and deploys the custom HTML5 app to the Cerbo GX. Overrides the stock Victron app with custom pages served via nginx. Supports SSH and USB installation. | +| [venus-html5-app](venus-html5-app/) | Frontend App | TypeScript/React | Fork of the Victron Venus HTML5 app with custom Marine2 views for weather, tides, tracking, generator status, and mooring. Displayed on Raymarine Axiom and other MFDs. | +| [watermaker](watermaker/) | UI + API Docs | React/JS | Control interface for a watermaker PLC system. React SPA with REST/WebSocket/MQTT integration. Backend runs on a separate PLC controller. | + +## Common D-Bus Service Structure + +All Python D-Bus services share this layout: + +``` +dbus-/ +├── .py # Main service entry point +├── config.py # Configuration constants +├── service/ +│ ├── run # daemontools entry point +│ └── log/run # multilog configuration +├── install.sh # Venus OS installation +├── uninstall.sh # Cleanup +├── build-package.sh # Creates deployable .tar.gz +└── README.md +``` + +At install time, `install.sh` symlinks `velib_python` from `/opt/victronenergy/`, registers the service with daemontools, and adds an `rc.local` entry for persistence across firmware updates. + +## Deployment + +```bash +# Build a package +cd dbus- +./build-package.sh + +# Copy to Cerbo GX +scp dbus--*.tar.gz root@:/data/ + +# Install on device +ssh root@ +cd /data && tar -xzf dbus--*.tar.gz +bash /data/dbus-/install.sh + +# Check service status +svstat /service/dbus- +tail -f /var/log/dbus-/current | tai64nlocal +``` + +## Development Prerequisites + +- **Python 3.8+** -- for all D-Bus services +- **Node.js 20+** and **npm** -- for venus-html5-app and watermaker UI +- **SSH/root access** to the Cerbo GX +- **Venus OS v3.10+** on the target device + +## Sensitive Configuration + +Some projects require API keys or credentials. These are excluded from version control. Copy the example templates and fill in your values: + +- `dbus-meteoblue-forecast/forecast_config.example.json` --> `forecast_config.json` +- `dbus-windy-station/station_config.example.json` --> `station_config.json` +- `watermaker/ui/.env.example` --> `.env` + +## New Project Template + +See [`_template/`](_template/) for a skeleton D-Bus service with all the boilerplate: main script, config, daemontools service, install/uninstall scripts, and build packaging. diff --git a/_template/.gitignore b/_template/.gitignore new file mode 100644 index 0000000..5923ff5 --- /dev/null +++ b/_template/.gitignore @@ -0,0 +1,24 @@ +# Build artifacts +*.tar.gz +*.sha256 + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg +*.egg-info/ +dist/ +build/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Venus OS runtime (created during installation) +ext/ diff --git a/_template/README.md b/_template/README.md new file mode 100644 index 0000000..d54826b --- /dev/null +++ b/_template/README.md @@ -0,0 +1,44 @@ +# dbus-template + +Venus OS D-Bus service template. Use this as a starting point for new services. + +## Getting Started + +1. Copy this directory: `cp -r _template/ dbus-your-service-name/` +2. Rename `dbus-template.py` to match your service (e.g. `your_service.py`) +3. Update `config.py` with your service name, product ID, and settings +4. Update `service/run` to point to your renamed main script +5. Update `install.sh`: set `SERVICE_NAME` and `MAIN_SCRIPT` variables +6. Update `build-package.sh`: set `PACKAGE_NAME` and the file copy list +7. Replace this README with your own documentation + +## Files + +| File | Purpose | +|------|---------| +| `dbus-template.py` | Main service with D-Bus registration boilerplate | +| `config.py` | Configuration constants (service name, product ID, timing) | +| `service/run` | daemontools entry point | +| `service/log/run` | multilog configuration | +| `install.sh` | Venus OS installation (velib symlink, service registration, rc.local) | +| `uninstall.sh` | Removes service symlink | +| `build-package.sh` | Creates a deployable .tar.gz package | + +## D-Bus Paths + +Update these for your service: + +| Path | Type | Description | +|------|------|-------------| +| `/ProductName` | string | Service display name | +| `/Connected` | int | Connection status (0/1) | +| `/Settings/Template/Enabled` | int | Enable/disable via settings | + +## Checklist + +- [ ] Unique service name in `config.py` (`com.victronenergy.yourservice`) +- [ ] Unique product ID in `config.py` (check existing services to avoid conflicts) +- [ ] All file references updated in `service/run`, `install.sh`, `build-package.sh` +- [ ] Custom D-Bus paths added +- [ ] Settings paths updated +- [ ] README replaced with your own documentation diff --git a/_template/build-package.sh b/_template/build-package.sh new file mode 100755 index 0000000..ff1b4a7 --- /dev/null +++ b/_template/build-package.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# +# Build script for Venus OS D-Bus service package +# +# Usage: +# ./build-package.sh +# ./build-package.sh --version 1.0.0 +# + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +VERSION="0.1.0" +OUTPUT_DIR="$SCRIPT_DIR" +PACKAGE_NAME="dbus-template" + +while [[ $# -gt 0 ]]; do + case $1 in + --version|-v) VERSION="$2"; shift 2 ;; + --output|-o) OUTPUT_DIR="$2"; shift 2 ;; + --help|-h) + echo "Usage: $0 [--version VERSION] [--output PATH]" + exit 0 + ;; + *) echo "Unknown option: $1"; exit 1 ;; + esac +done + +BUILD_DIR=$(mktemp -d) +PACKAGE_DIR="$BUILD_DIR/$PACKAGE_NAME" + +echo "Building $PACKAGE_NAME v$VERSION..." + +mkdir -p "$PACKAGE_DIR/service/log" + +# Copy application files -- update this list for your service +cp "$SCRIPT_DIR/dbus-template.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/config.py" "$PACKAGE_DIR/" + +# Copy service and install files +cp "$SCRIPT_DIR/service/run" "$PACKAGE_DIR/service/" +cp "$SCRIPT_DIR/service/log/run" "$PACKAGE_DIR/service/log/" +cp "$SCRIPT_DIR/install.sh" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/uninstall.sh" "$PACKAGE_DIR/" + +# Set permissions +chmod +x "$PACKAGE_DIR/dbus-template.py" +chmod +x "$PACKAGE_DIR/install.sh" +chmod +x "$PACKAGE_DIR/uninstall.sh" +chmod +x "$PACKAGE_DIR/service/run" +chmod +x "$PACKAGE_DIR/service/log/run" + +# Create archive +mkdir -p "$OUTPUT_DIR" +TARBALL="$PACKAGE_NAME-$VERSION.tar.gz" +OUTPUT_ABS="$(cd "$OUTPUT_DIR" && pwd)" +cd "$BUILD_DIR" +tar --format=ustar -czf "$OUTPUT_ABS/$TARBALL" "$PACKAGE_NAME" +rm -rf "$BUILD_DIR" + +echo "Package: $OUTPUT_ABS/$TARBALL" +echo "" +echo "Install on Venus OS:" +echo " scp $OUTPUT_ABS/$TARBALL root@:/data/" +echo " ssh root@" +echo " cd /data && tar -xzf $TARBALL" +echo " bash /data/$PACKAGE_NAME/install.sh" diff --git a/_template/config.py b/_template/config.py new file mode 100644 index 0000000..1d28db3 --- /dev/null +++ b/_template/config.py @@ -0,0 +1,22 @@ +""" +Configuration for dbus-YOUR-SERVICE-NAME. + +Rename this file is not needed -- just update the values below. +""" + +# Service identity +SERVICE_NAME = 'com.victronenergy.yourservice' +DEVICE_INSTANCE = 0 +PRODUCT_NAME = 'Your Service Name' +PRODUCT_ID = 0xA1FF # Pick a unique product ID (0xA100-0xA1FF range) +FIRMWARE_VERSION = 0 +CONNECTED = 1 + +# Version +VERSION = '0.1.0' + +# Timing +MAIN_LOOP_INTERVAL_MS = 1000 # Main loop tick (milliseconds) + +# Logging +LOG_LEVEL = 'INFO' # DEBUG, INFO, WARNING, ERROR diff --git a/_template/dbus-template.py b/_template/dbus-template.py new file mode 100755 index 0000000..1832a75 --- /dev/null +++ b/_template/dbus-template.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +""" +Venus OS D-Bus service template. + +To create a new service: + 1. Copy the _template/ directory and rename it to dbus-/ + 2. Rename this file to match your service (e.g. your_service.py) + 3. Update config.py with your service name, product ID, etc. + 4. Update service/run to point to your renamed script + 5. Update install.sh with your service name + 6. Update build-package.sh with your file list +""" + +import logging +import os +import signal +import sys +import time + +sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'ext', 'velib_python')) + +from vedbus import VeDbusService # noqa: E402 +from settingsdevice import SettingsDevice # noqa: E402 +import dbus # noqa: E402 +from gi.repository import GLib # noqa: E402 + +from config import ( # noqa: E402 + SERVICE_NAME, + DEVICE_INSTANCE, + PRODUCT_NAME, + PRODUCT_ID, + FIRMWARE_VERSION, + CONNECTED, + MAIN_LOOP_INTERVAL_MS, + LOG_LEVEL, + VERSION, +) + +logger = logging.getLogger('dbus-template') + + +class TemplateService: + """Main service class. Rename to match your service.""" + + def __init__(self): + self._running = False + self._dbusservice = None + self._settings = None + + def start(self): + """Initialize D-Bus service and start the main loop.""" + bus = dbus.SystemBus() + + self._dbusservice = VeDbusService( + SERVICE_NAME, bus=bus, register=False + ) + + # Mandatory D-Bus paths + self._dbusservice.add_path('/Mgmt/ProcessName', __file__) + self._dbusservice.add_path('/Mgmt/ProcessVersion', VERSION) + self._dbusservice.add_path('/Mgmt/Connection', 'local') + self._dbusservice.add_path('/DeviceInstance', DEVICE_INSTANCE) + self._dbusservice.add_path('/ProductId', PRODUCT_ID) + self._dbusservice.add_path('/ProductName', PRODUCT_NAME) + self._dbusservice.add_path('/FirmwareVersion', FIRMWARE_VERSION) + self._dbusservice.add_path('/Connected', CONNECTED) + + # --- Add your custom D-Bus paths here --- + # self._dbusservice.add_path('/YourPath', initial_value) + + # Settings (stored in Venus OS localsettings, persist across reboots) + settings_path = '/Settings/Template' + supported_settings = { + 'enabled': [settings_path + '/Enabled', 1, 0, 1], + # 'your_setting': [settings_path + '/YourSetting', default, min, max], + } + self._settings = SettingsDevice( + bus, supported_settings, self._on_setting_changed + ) + + self._dbusservice.register() + logger.info('Service registered on D-Bus as %s', SERVICE_NAME) + + self._running = True + GLib.timeout_add(MAIN_LOOP_INTERVAL_MS, self._update) + + def _update(self): + """Called every MAIN_LOOP_INTERVAL_MS. Return True to keep running.""" + if not self._running: + return False + + # --- Add your main loop logic here --- + + return True + + def _on_setting_changed(self, setting, old, new): + """Called when a Venus OS setting changes.""" + logger.info('Setting %s changed: %s -> %s', setting, old, new) + + def stop(self): + """Clean shutdown.""" + self._running = False + logger.info('Service stopped') + + +def main(): + logging.basicConfig( + level=getattr(logging, LOG_LEVEL, logging.INFO), + format='%(asctime)s %(name)s %(levelname)s %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + ) + logger.info('Starting dbus-template v%s', VERSION) + + service = TemplateService() + + def shutdown(signum, frame): + logger.info('Received signal %d, shutting down...', signum) + service.stop() + sys.exit(0) + + signal.signal(signal.SIGTERM, shutdown) + signal.signal(signal.SIGINT, shutdown) + + service.start() + + mainloop = GLib.MainLoop() + mainloop.run() + + +if __name__ == '__main__': + main() diff --git a/_template/install.sh b/_template/install.sh new file mode 100755 index 0000000..8613ab2 --- /dev/null +++ b/_template/install.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# +# Installation script for Venus OS D-Bus service template +# +# Usage: +# chmod +x install.sh +# ./install.sh +# + +set -e + +SERVICE_NAME="dbus-template" +INSTALL_DIR="/data/$SERVICE_NAME" +MAIN_SCRIPT="dbus-template.py" + +# Find velib_python +VELIB_DIR="" +if [ -d "/opt/victronenergy/velib_python" ]; then + VELIB_DIR="/opt/victronenergy/velib_python" +else + for candidate in \ + "/opt/victronenergy/dbus-systemcalc-py/ext/velib_python" \ + "/opt/victronenergy/dbus-generator/ext/velib_python" \ + "/opt/victronenergy/dbus-mqtt/ext/velib_python" \ + "/opt/victronenergy/dbus-digitalinputs/ext/velib_python" \ + "/opt/victronenergy/vrmlogger/ext/velib_python" + do + if [ -d "$candidate" ] && [ -f "$candidate/vedbus.py" ]; then + VELIB_DIR="$candidate" + break + fi + done +fi + +if [ -z "$VELIB_DIR" ]; then + VEDBUS_PATH=$(find /opt/victronenergy -name "vedbus.py" -path "*/velib_python/*" 2>/dev/null | head -1) + if [ -n "$VEDBUS_PATH" ]; then + VELIB_DIR=$(dirname "$VEDBUS_PATH") + fi +fi + +# Determine service directory +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +elif [ -L "/service" ]; then + SERVICE_DIR=$(readlink -f /service) +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "==================================================" +echo "$SERVICE_NAME - Installation" +echo "==================================================" + +if [ ! -d "$SERVICE_DIR" ]; then + echo "ERROR: This doesn't appear to be a Venus OS device." + exit 1 +fi + +if [ ! -f "$INSTALL_DIR/$MAIN_SCRIPT" ]; then + echo "ERROR: Installation files not found in $INSTALL_DIR" + exit 1 +fi + +echo "1. Making scripts executable..." +chmod +x "$INSTALL_DIR/service/run" +chmod +x "$INSTALL_DIR/service/log/run" +chmod +x "$INSTALL_DIR/$MAIN_SCRIPT" + +echo "2. Creating velib_python symlink..." +if [ -z "$VELIB_DIR" ]; then + echo "ERROR: Could not find velib_python on this system." + exit 1 +fi +echo " Found velib_python at: $VELIB_DIR" +mkdir -p "$INSTALL_DIR/ext" +if [ -L "$INSTALL_DIR/ext/velib_python" ]; then + rm "$INSTALL_DIR/ext/velib_python" +fi +ln -s "$VELIB_DIR" "$INSTALL_DIR/ext/velib_python" + +echo "3. Creating service symlink..." +if [ -L "$SERVICE_DIR/$SERVICE_NAME" ] || [ -e "$SERVICE_DIR/$SERVICE_NAME" ]; then + rm -rf "$SERVICE_DIR/$SERVICE_NAME" +fi +ln -s "$INSTALL_DIR/service" "$SERVICE_DIR/$SERVICE_NAME" + +echo "4. Creating log directory..." +mkdir -p "/var/log/$SERVICE_NAME" + +echo "5. Setting up rc.local for persistence..." +RC_LOCAL="/data/rc.local" +if [ ! -f "$RC_LOCAL" ]; then + echo "#!/bin/bash" > "$RC_LOCAL" + chmod +x "$RC_LOCAL" +fi + +if ! grep -q "$SERVICE_NAME" "$RC_LOCAL"; then + echo "" >> "$RC_LOCAL" + echo "# $SERVICE_NAME" >> "$RC_LOCAL" + echo "if [ ! -L $SERVICE_DIR/$SERVICE_NAME ]; then" >> "$RC_LOCAL" + echo " ln -s /data/$SERVICE_NAME/service $SERVICE_DIR/$SERVICE_NAME" >> "$RC_LOCAL" + echo "fi" >> "$RC_LOCAL" +fi + +echo "" +echo "==================================================" +echo "Installation complete!" +echo "==================================================" +echo "" +echo "To check status: svstat $SERVICE_DIR/$SERVICE_NAME" +echo "To view logs: tail -F /var/log/$SERVICE_NAME/current | tai64nlocal" +echo "" diff --git a/_template/service/log/run b/_template/service/log/run new file mode 100755 index 0000000..a636573 --- /dev/null +++ b/_template/service/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec multilog t s99999 n8 /var/log/dbus-template diff --git a/_template/service/run b/_template/service/run new file mode 100755 index 0000000..66843f2 --- /dev/null +++ b/_template/service/run @@ -0,0 +1,5 @@ +#!/bin/sh +exec 2>&1 +cd /data/dbus-template +export PYTHONPATH="/data/dbus-template/ext/velib_python:$PYTHONPATH" +exec python3 /data/dbus-template/dbus-template.py diff --git a/_template/uninstall.sh b/_template/uninstall.sh new file mode 100755 index 0000000..e645e56 --- /dev/null +++ b/_template/uninstall.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Uninstall script for Venus OS D-Bus service template +# + +set -e + +SERVICE_NAME="dbus-template" + +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +elif [ -L "/service" ]; then + SERVICE_DIR=$(readlink -f /service) +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "Uninstalling $SERVICE_NAME..." + +if [ -L "$SERVICE_DIR/$SERVICE_NAME" ]; then + svc -d "$SERVICE_DIR/$SERVICE_NAME" 2>/dev/null || true + sleep 2 + rm "$SERVICE_DIR/$SERVICE_NAME" + echo "Service symlink removed" +fi + +echo "" +echo "Uninstall complete." +echo "Files remain in /data/$SERVICE_NAME/ -- remove manually if desired." +echo "" diff --git a/axiom-nmea/.gitignore b/axiom-nmea/.gitignore new file mode 100644 index 0000000..d7a3332 --- /dev/null +++ b/axiom-nmea/.gitignore @@ -0,0 +1,24 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg-info/ +dist/ +build/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# Packet captures (keep samples/ dir but ignore pcap files within) +captures/ +samples/*.pcap +*.bin + +# Environment +.env +venv/ diff --git a/axiom-nmea/PROTOCOL.md b/axiom-nmea/PROTOCOL.md new file mode 100644 index 0000000..142766e --- /dev/null +++ b/axiom-nmea/PROTOCOL.md @@ -0,0 +1,624 @@ +# Raymarine LightHouse Protocol Analysis + +## Overview + +This document describes the findings from reverse-engineering the Raymarine LightHouse network protocol used by AXIOM MFDs to share sensor data over IP multicast. + +**Key Discovery**: Raymarine does NOT use standard NMEA 0183 text sentences on its multicast network. Instead, it uses **Google Protocol Buffers** (protobuf) binary encoding over UDP multicast. + +## Quick Reference + +### Decoding Status Summary + +| Sensor | Status | Field | Unit | +|--------|--------|-------|------| +| GPS Position | ✅ Reliable | 2.1, 2.2 | Decimal degrees | +| **SOG (Speed Over Ground)** | ✅ Reliable | 5.5 | m/s → knots | +| **COG (Course Over Ground)** | ✅ Reliable | 5.1 | Radians → degrees | +| Compass Heading | ⚠️ Variable | 3.2 | Radians → degrees | +| Wind Direction | ⚠️ Variable | 13.4 | Radians → degrees | +| Wind Speed | ⚠️ Variable | 13.5, 13.6 | m/s → knots | +| Depth | ⚠️ Variable | 7.1 | Meters → feet | +| Barometric Pressure | ✅ Reliable | 15.1 | Pascals → mbar | +| Water Temperature | ✅ Reliable | 15.9 | Kelvin → Celsius | +| Air Temperature | ⚠️ Variable | 15.3 | Kelvin → Celsius | +| Tank Levels | ✅ Reliable | 16 | Percentage (0-100%) | +| House Batteries | ✅ Reliable | 20 | Volts (direct) | +| Engine Batteries | ✅ Reliable | 14.3.4 | Volts (direct) | + +### Primary Data Source +- **Multicast:** `226.192.206.102:2565` +- **Source IP:** `198.18.1.170` (AXIOM 12 Data Master) +- **Packet Format:** 20-byte header + Protocol Buffers payload + +--- + +## Network Configuration + +### Multicast Groups + +| Group | Port | Source IP | Device | Purpose | +|-------|------|-----------|--------|---------| +| 226.192.206.98 | 2561 | 10.22.6.115 | Unknown | Navigation (mostly zeros) | +| 226.192.206.99 | 2562 | 198.18.1.170 | AXIOM 12 Data Master | Heartbeat/status | +| 226.192.206.102 | 2565 | 198.18.1.170 | AXIOM 12 Data Master | **Primary sensor data** | +| 226.192.219.0 | 3221 | 198.18.2.191 | AXIOM PLUS 12 RV | Display synchronization | + +Additional groups that may contain sensor/tank data: +- `226.192.206.100:2563` +- `226.192.206.101:2564` +- `239.2.1.1:2154` + +### Data Sources + +| IP Address | Ports | Device | Data Types | +|------------|-------|--------|------------| +| 198.18.1.170 | 35044, 41741 | AXIOM 12 (Data Master) | GPS, Wind, Depth, Heading, Temp, Tanks, Batteries | +| 198.18.2.191 | 35022, 45403, 50194 | AXIOM PLUS 12 RV | Display sync, possible depth relay | +| 10.22.6.115 | 57601 | Unknown | Mostly zero values | + +### Packet Sizes + +The data master (198.18.1.170) sends packets of varying sizes: + +| Size (bytes) | Frequency | Contents | +|--------------|-----------|----------| +| 16 | Low | Minimal/heartbeat | +| 54 | Low | Short messages | +| 91-92 | Medium | Status/heartbeat | +| 344 | Medium | Partial sensor data | +| 446 | Medium | Sensor data | +| 788-903 | Medium | Extended sensor data | +| 1003 | Medium | Extended sensor data | +| 1810-2056 | High | Full navigation data including GPS | + +--- + +## Packet Structure + +### Fixed Header (20 bytes) + +All packets begin with a 20-byte fixed header before the protobuf payload: + +``` +Offset Size Description +------ ---- ----------- +0x0000 8 Packet identifier (00 00 00 00 00 00 00 01) +0x0008 4 Source ID +0x000C 4 Message type indicator +0x0010 4 Payload length +``` + +**Protobuf payload starts at offset 0x14 (20 decimal).** + +### Protobuf Message Structure + +The payload uses Google Protocol Buffers wire format. Top-level fields: + +``` +Field 1 (length) - Device Info (name, serial number) +Field 2 (length) - GPS/Position Data + ├─ Field 1 (fixed64/double) - Latitude + └─ Field 2 (fixed64/double) - Longitude + +Field 3 (length) - Heading Block + └─ Field 2 (fixed32/float) - Heading (radians) + +Field 5 (length) - SOG/COG Navigation Data (86-92 byte packets) + ├─ Field 1 (fixed32/float) - COG Course Over Ground (radians) + ├─ Field 3 (fixed32/float) - Unknown constant (0.05) + ├─ Field 4 (fixed32/float) - Unknown constant (0.1) + ├─ Field 5 (fixed32/float) - SOG Speed Over Ground (m/s) + ├─ Field 6 (fixed32/float) - Secondary angle (radians) - possibly heading + └─ Field 7 (fixed32/float) - Unknown constant (11.93) + +Field 7 (length) - Depth Block (large packets only) + └─ Field 1 (fixed32/float) - Depth (meters) + +Field 13 (length) - Wind/Navigation Data + ├─ Field 4 (fixed32/float) - True Wind Direction (radians) + ├─ Field 5 (fixed32/float) - True Wind Speed (m/s) + └─ Field 6 (fixed32/float) - Apparent Wind Speed (m/s) + +Field 14 (repeated) - Engine Data + ├─ Field 1 (varint) - Engine ID (0=Port, 1=Starboard) + └─ Field 3 (length) - Engine Sensor Data + └─ Field 4 (fixed32/float) - Battery Voltage (volts) + +Field 15 (length) - Environment Data + ├─ Field 1 (fixed32/float) - Barometric Pressure (Pascals) + ├─ Field 3 (fixed32/float) - Air Temperature (Kelvin) + └─ Field 9 (fixed32/float) - Water Temperature (Kelvin) + +Field 16 (repeated) - Tank Data + ├─ Field 1 (varint) - Tank ID + ├─ Field 2 (varint) - Status/Flag + └─ Field 3 (fixed32/float) - Tank Level (percentage) + +Field 20 (repeated) - House Battery Data + ├─ Field 1 (varint) - Battery ID (11=Aft, 13=Stern) + └─ Field 3 (fixed32/float) - Voltage (volts) +``` + +--- + +## Wire Format Details + +Raymarine uses Protocol Buffers with these wire types: + +| Wire Type | Name | Size | Usage | +|-----------|------|------|-------| +| 0 | Varint | Variable | IDs, counts, enums, status flags | +| 1 | Fixed64 | 8 bytes | High-precision values (GPS coordinates) | +| 2 | Length-delimited | Variable | Nested messages, byte strings | +| 5 | Fixed32 | 4 bytes | Floats (angles, speeds, voltages) | + +### Tag Format +Each field is prefixed by a tag byte: `(field_number << 3) | wire_type` + +Examples: +- `0x09` = Field 1, wire type 1 (fixed64) +- `0x11` = Field 2, wire type 1 (fixed64) +- `0x15` = Field 2, wire type 5 (fixed32) +- `0x1d` = Field 3, wire type 5 (fixed32) + +### Unit Conventions + +| Measurement | Raw Unit | Conversion | +|-------------|----------|------------| +| Latitude/Longitude | Decimal degrees | Direct (WGS84) | +| Angles (heading, wind) | Radians | × 57.2957795131 = degrees | +| Wind speed | m/s | × 1.94384449 = knots | +| Depth | Meters | ÷ 0.3048 = feet | +| Temperature | Kelvin | − 273.15 = Celsius | +| Barometric Pressure | Pascals | × 0.01 = mbar (hPa) | +| Tank levels | Percentage | 0-100% direct | +| Voltage | Volts | Direct value | + +--- + +## Field Extraction Methods + +### GPS Position ✅ RELIABLE + +**Location:** Field 2.1 (latitude), Field 2.2 (longitude) + +```python +# Parse Field 2 as nested message, then extract: +Field 2.1 (fixed64/double) → Latitude in decimal degrees +Field 2.2 (fixed64/double) → Longitude in decimal degrees + +# Validation +-90 ≤ latitude ≤ 90 +-180 ≤ longitude ≤ 180 +abs(lat) > 0.1 or abs(lon) > 0.1 # Not at null island +``` + +**Example decode:** +``` +Hex: 09 cf 20 f4 22 c9 ee 38 40 11 b4 6f 93 f6 2b 28 54 c0 + | | | | + | +-- Latitude double | +-- Longitude double + +-- Field 1 tag +-- Field 2 tag + +Latitude: 24.932757° N +Longitude: -80.627683° W +``` + +--- + +### SOG (Speed Over Ground) ✅ RELIABLE + +**Location:** Field 5.5 + +```python +Field 5.5 (fixed32/float) → SOG in meters per second + +# Conversion +sog_knots = sog_ms × 1.94384449 + +# Validation +0 ≤ sog ≤ 50 (m/s, roughly 0-100 knots) +``` + +**Notes:** +- Found in 86-92 byte packets +- At dock, value is near zero (~0.01 m/s = 0.02 kts) +- Derived from GPS, so requires GPS lock + +--- + +### COG (Course Over Ground) ✅ RELIABLE + +**Location:** Field 5.1 + +```python +Field 5.1 (fixed32/float) → COG in radians + +# Conversion +cog_degrees = (radians × 57.2957795131) % 360 + +# Validation +0 ≤ radians ≤ 6.5 (approximately 0 to 2π) +``` + +**Notes:** +- Found in 86-92 byte packets +- At dock/low speed, COG jumps randomly (GPS noise when stationary) +- Field 5.6 also contains an angle that varies similarly (possibly heading-from-GPS) + +--- + +### Field 5 Complete Structure + +| Subfield | Wire Type | Purpose | Notes | +|----------|-----------|---------|-------| +| 5 | f64 | Unknown | Often zero | +| **5.1** | f32 | **COG** (radians) | Course Over Ground | +| 5.3 | f32 | Unknown | Constant 0.05 | +| 5.4 | f32 | Unknown | Constant 0.1 | +| **5.5** | f32 | **SOG** (m/s) | Speed Over Ground | +| 5.6 | f32 | Secondary angle | Varies like COG | +| 5.7 | f32 | Unknown | Constant 11.93 | + +--- + +### Compass Heading ⚠️ VARIABLE + +**Location:** Field 3.2 + +```python +Field 3.2 (fixed32/float) → Heading in radians + +# Conversion +heading_degrees = radians × 57.2957795131 +heading_degrees = heading_degrees % 360 + +# Validation +0 ≤ radians ≤ 6.5 (approximately 0 to 2π) +``` + +--- + +### Wind Data ⚠️ VARIABLE + +**Location:** Field 13.4, 13.5, 13.6 + +```python +Field 13.4 (fixed32/float) → True Wind Direction (radians) +Field 13.5 (fixed32/float) → True Wind Speed (m/s) +Field 13.6 (fixed32/float) → Apparent Wind Speed (m/s) + +# Conversions +direction_deg = radians × 57.2957795131 +speed_kts = speed_ms × 1.94384449 + +# Validation +0 ≤ angle ≤ 6.5 (radians) +0 ≤ speed ≤ 100 (m/s) +``` + +--- + +### Depth ⚠️ VARIABLE + +**Location:** Field 7.1 (only in larger packets 1472B+) + +```python +Field 7.1 (fixed32/float) → Depth in meters + +# Conversion +depth_feet = depth_meters / 0.3048 + +# Validation +0 < depth ≤ 1000 (meters) +``` + +--- + +### Barometric Pressure ✅ RELIABLE + +**Location:** Field 15.1 + +```python +Field 15.1 (fixed32/float) → Barometric Pressure (Pascals) + +# Conversion +pressure_mbar = pressure_pa * 0.01 +pressure_inhg = pressure_mbar * 0.02953 + +# Validation +87000 ≤ value ≤ 108400 Pa (870-1084 mbar) +``` + +--- + +### Temperature ✅ RELIABLE + +**Location:** Field 15.3 (air), Field 15.9 (water) + +```python +Field 15.3 (fixed32/float) → Air Temperature (Kelvin) +Field 15.9 (fixed32/float) → Water Temperature (Kelvin) + +# Conversion +temp_celsius = temp_kelvin - 273.15 +temp_fahrenheit = temp_celsius × 9/5 + 32 + +# Validation +Air: 200 ≤ value ≤ 350 K (-73°C to 77°C) +Water: 270 ≤ value ≤ 320 K (-3°C to 47°C) +``` + +--- + +### Tank Levels ✅ RELIABLE + +**Location:** Field 16 (repeated) + +```python +Field 16 (repeated messages): + Field 1 (varint) → Tank ID + Field 2 (varint) → Status flag + Field 3 (fixed32/float) → Level percentage + +# Validation +0 ≤ level ≤ 100 (percentage) +``` + +**Tank ID Mapping:** + +| ID | Name | Capacity | Notes | +|----|------|----------|-------| +| 1 | Starboard Fuel | 265 gal | Has explicit ID | +| 2 | Port Fuel | 265 gal | Inferred (no ID, no status) | +| 10 | Forward Water | 90 gal | | +| 11 | Aft Water | 90 gal | | +| 100 | Black Water | 53 gal | Inferred (status=5) | + +**Inference Logic:** +```python +if tank_id is None: + if status == 5: + tank_id = 100 # Black/waste water + elif status is None: + tank_id = 2 # Port Fuel (only tank with no ID or status) +``` + +--- + +### House Batteries ✅ RELIABLE + +**Location:** Field 20 (repeated) + +```python +Field 20 (repeated messages): + Field 1 (varint) → Battery ID + Field 3 (fixed32/float) → Voltage (volts) + +# Validation +10 ≤ voltage ≤ 60 (covers 12V, 24V, 48V systems) +``` + +**Battery ID Mapping:** + +| ID | Name | Expected Voltage | +|----|------|------------------| +| 11 | Aft House | ~26.3V (24V system) | +| 13 | Stern House | ~27.2V (24V system) | + +--- + +### Engine Batteries ✅ RELIABLE + +**Location:** Field 14.3.4 (deep nested - 3 levels) + +```python +Field 14 (repeated messages): + Field 1 (varint) → Engine ID (0=Port, 1=Starboard) + Field 3 (length/nested message): + Field 4 (fixed32/float) → Battery voltage (volts) + +# Extraction requires parsing Field 14.3 as nested protobuf +# to extract Field 4 (voltage) + +# Battery ID calculation +battery_id = 1000 + engine_id +# Port Engine = 1000, Starboard Engine = 1001 + +# Validation +10 ≤ voltage ≤ 60 (volts) +``` + +**Engine Battery Mapping:** + +| Engine ID | Battery ID | Name | +|-----------|------------|------| +| 0 | 1000 | Port Engine | +| 1 | 1001 | Starboard Engine | + +--- + +## Technical Challenges + +### 1. No Schema Available + +Protocol Buffers normally use a `.proto` schema file to define message structure. Without Raymarine's proprietary schema, we cannot: +- Know message type identifiers +- Understand field semantics +- Differentiate between message types + +### 2. Field Number Collision + +The same protobuf field number means different things in different message types: +- Field 4 at one offset might be wind speed +- Field 4 at another offset might be something else entirely + +### 3. Variable Packet Structure + +Packets of different sizes have completely different internal layouts: +- GPS appears at offset ~0x0032 in large packets +- Sensor data appears at different offsets depending on packet size +- Nested submessages add complexity + +### 4. No Message Type Markers + +Unlike some protocols, there's no obvious message type identifier in the packet header that would allow us to switch parsing logic based on message type. + +### 5. Mixed Precision + +Some values use 64-bit doubles, others use 32-bit floats. Both can appear in the same packet, and the same logical value (e.g., an angle) might be encoded differently in different message types. + +--- + +## Recommended Approach for Reliable Decoding + +### Option 1: GPS-Anchored Parsing + +1. Find GPS using the reliable `0x09`/`0x11` pattern +2. Use GPS offset as anchor point +3. Extract values at fixed byte offsets relative to GPS +4. Maintain separate offset tables for each packet size + +### Option 2: Packet Size Dispatch + +1. Identify packet by size +2. Apply size-specific parsing rules +3. Use absolute byte offsets (not field numbers) +4. Maintain a mapping table: `(packet_size, offset) → sensor_type` + +### Option 3: Value Correlation + +1. Collect all extracted values +2. Compare against known ground truth (displayed values on MFD) +3. Use statistical correlation to identify correct mappings +4. Build confidence scores for each mapping + +--- + +## Tools Included + +### Main Decoders + +| Tool | Purpose | +|------|---------| +| `protobuf_decoder.py` | **Primary decoder** - all fields via proper protobuf parsing | +| `raymarine_decoder.py` | High-level decoder with live dashboard display | + +### Discovery & Debug Tools + +| Tool | Purpose | +|------|---------| +| `battery_debug.py` | Deep nesting parser for Field 14.3.4 (engine batteries) | +| `battery_finder.py` | Scans multicast groups for voltage-like values | +| `tank_debug.py` | Raw Field 16 entry inspection | +| `tank_finder.py` | Searches for tank level percentages | +| `field_debugger.py` | Deep analysis of packet fields | + +### Analysis Tools + +| Tool | Purpose | +|------|---------| +| `analyze_structure.py` | Packet structure analysis | +| `field_mapping.py` | Documents the protobuf structure | +| `protobuf_parser.py` | Lower-level wire format decoder | +| `watch_field.py` | Monitor specific field values over time | + +### Wind/Heading Finders + +| Tool | Purpose | +|------|---------| +| `wind_finder.py` | Searches for wind speed values | +| `find_twd.py` | Searches for true wind direction | +| `find_heading_vs_twd.py` | Compares heading and TWD values | +| `find_consistent_heading.py` | Identifies stable heading fields | + +## Usage + +```bash +# Run the primary protobuf decoder (live network) +python protobuf_decoder.py -i YOUR_VLAN_IP + +# JSON output for integration +python protobuf_decoder.py -i YOUR_VLAN_IP --json + +# Decode from pcap file (offline analysis) +python protobuf_decoder.py --pcap raymarine_sample.pcap + +# Debug battery extraction +python battery_debug.py --pcap raymarine_sample.pcap + +# Debug tank data +python tank_debug.py --pcap raymarine_sample.pcap +``` + +Replace `YOUR_VLAN_IP` with your interface IP on the Raymarine VLAN (e.g., `198.18.5.5`). + +**No external dependencies required** - uses only Python standard library. + +--- + +## Sample Output + +``` +============================================================ + RAYMARINE DECODER (Protobuf) 17:36:01 +============================================================ + GPS: 24.932652, -80.627569 + Heading: 35.2° + Wind: 14.6 kts @ 68.5° (true) + Depth: 7.5 ft (2.3 m) + Temp: Air 24.8°C / 76.6°F, Water 26.2°C / 79.2°F + Tanks: Stbd Fuel: 75.2% (199gal), Port Fuel: 68.1% (180gal), ... + Batts: Aft House: 26.3V, Stern House: 27.2V, Port Engine: 26.5V +------------------------------------------------------------ + Packets: 4521 Decoded: 4312 Uptime: 85.2s +============================================================ +``` + +### JSON Output + +```json +{ + "timestamp": "2025-12-23T17:36:01.123456", + "position": {"latitude": 24.932652, "longitude": -80.627569}, + "navigation": {"heading_deg": 35.2, "cog_deg": null, "sog_kts": null}, + "wind": {"true_direction_deg": 68.5, "true_speed_kts": 14.6, ...}, + "depth": {"feet": 7.5, "meters": 2.3}, + "temperature": {"water_c": 26.2, "air_c": 24.8}, + "tanks": { + "1": {"name": "Stbd Fuel", "level_pct": 75.2, "capacity_gal": 265}, + "2": {"name": "Port Fuel", "level_pct": 68.1, "capacity_gal": 265} + }, + "batteries": { + "11": {"name": "Aft House", "voltage_v": 26.3}, + "13": {"name": "Stern House", "voltage_v": 27.2}, + "1000": {"name": "Port Engine", "voltage_v": 26.5} + } +} +``` + +--- + +## Future Work + +1. ~~**SOG/COG extraction**~~ ✅ **DONE** - Field 5.5 (SOG) and Field 5.1 (COG) identified +2. **Apparent Wind Angle** - AWA field location to be confirmed +3. **Additional engine data** - RPM, fuel flow, oil pressure likely in Field 14 +4. **Field 5.3, 5.4, 5.7** - Unknown constants (0.05, 0.1, 11.93) - purpose TBD +5. **Investigate SignalK** - The MFDs expose HTTP on port 8080 which may provide a cleaner API +6. **NMEA TCP/UDP** - Check if standard NMEA is available on other ports (10110, 2000, etc.) + +--- + +## References + +- [Protocol Buffers Encoding](https://developers.google.com/protocol-buffers/docs/encoding) +- [Raymarine LightHouse OS](https://www.raymarine.com/lighthouse/) +- Test location: Florida Keys (24° 55' N, 80° 37' W) + +--- + +## License + +This reverse-engineering effort is for personal/educational use. The Raymarine protocol is proprietary. diff --git a/axiom-nmea/README.md b/axiom-nmea/README.md new file mode 100644 index 0000000..3863da0 --- /dev/null +++ b/axiom-nmea/README.md @@ -0,0 +1,194 @@ +# Raymarine LightHouse Protocol Decoder + +A Python toolkit for decoding sensor data from Raymarine AXIOM/LightHouse multicast networks. + +> **See [PROTOCOL.md](PROTOCOL.md) for detailed protocol analysis and field mappings.** + +## Protocol Discovery Summary + +**Key Finding: Raymarine uses Google Protocol Buffers over UDP multicast, NOT standard NMEA 0183.** + +### Decoding Status + +| Sensor | Status | Field | Notes | +|--------|--------|-------|-------| +| **GPS Position** | Reliable | 2.1, 2.2 | 64-bit doubles, decimal degrees | +| Compass Heading | Variable | 3.2 | 32-bit float, radians | +| Wind Direction | Variable | 13.4 | 32-bit float, radians (true) | +| Wind Speed | Variable | 13.5, 13.6 | 32-bit float, m/s (true/apparent) | +| Depth | Variable | 7.1 | 32-bit float, meters | +| **Water Temperature** | Reliable | 15.9 | 32-bit float, Kelvin | +| Air Temperature | Variable | 15.3 | 32-bit float, Kelvin | +| **Tank Levels** | Reliable | 16 | Repeated field with ID, status, level % | +| **House Batteries** | Reliable | 20 | Repeated field with ID and voltage | +| **Engine Batteries** | Reliable | 14.3.4 | Deep nested (3 levels) with voltage | + +### Why Variable? + +Without Raymarine's proprietary protobuf schema, we're reverse-engineering blind: +- Same field numbers mean different things in different packet types +- Packet structure varies by size (344 bytes vs 2056 bytes) +- No message type identifiers in headers + +See [PROTOCOL.md](PROTOCOL.md) for the full technical analysis. + +### Multicast Groups & Sources + +| Group | Port | Source IP | Device | Data | +|-------|------|-----------|--------|------| +| 226.192.206.98 | 2561 | 10.22.6.115 | Unknown | Navigation (mostly zeros) | +| 226.192.206.99 | 2562 | 198.18.1.170 | AXIOM 12 (Data Master) | Heartbeat/status | +| 226.192.206.102 | 2565 | 198.18.1.170 | AXIOM 12 (Data Master) | **Primary sensor data** | +| 226.192.219.0 | 3221 | 198.18.2.191 | AXIOM PLUS 12 RV | Display sync | + +**Primary Data Source:** `198.18.1.170` broadcasts GPS, wind, depth, heading, temperatures, tank levels, and battery voltages. + +### Data Encoding +- Wire format: Protobuf (fixed64 doubles + fixed32 floats) +- Angles: **Radians** (multiply by 57.2958 for degrees) +- Wind speed: **m/s** (multiply by 1.94384 for knots) +- Depth: **Meters** (divide by 0.3048 for feet) +- Temperature: **Kelvin** (subtract 273.15 for Celsius) +- Tank levels: **Percentage** (0-100%) +- Battery voltage: **Volts** (direct value) + +## Quick Start + +**No installation required** - clone and run. Uses only Python standard library. + +```bash +git clone https://github.com/terbonium/axiom-nmea.git +cd axiom-nmea +``` + +Optional: Install as a package for use in your own projects: +```bash +pip install -e . +``` + +## Usage + +```bash +# Run the decoder with live dashboard (requires network access to Raymarine VLAN) +python debug/protobuf_decoder.py -i YOUR_VLAN_IP + +# JSON output (for integration with other systems) +python debug/protobuf_decoder.py -i YOUR_VLAN_IP --json + +# Decode from pcap file (offline analysis) +python debug/protobuf_decoder.py --pcap samples/raymarine_sample.pcap +``` + +Replace `YOUR_VLAN_IP` with your VLAN interface IP (e.g., `198.18.5.5`). + +## Sample Output + +``` +============================================================ + RAYMARINE DECODER (Protobuf) 16:13:21 +============================================================ + GPS: 24.932652, -80.627569 + Heading: 35.2° + Wind: 14.6 kts @ 68.5° (true) + Depth: 7.5 ft (2.3 m) + Temp: Air 24.8°C / 76.6°F, Water 26.2°C / 79.2°F + Tanks: Stbd Fuel: 75.2% (199gal), Port Fuel: 68.1% (180gal), ... + Batts: Aft House: 26.3V, Stern House: 27.2V, Port Engine: 26.5V +------------------------------------------------------------ + Packets: 9444 Decoded: 8921 Uptime: 124.5s +============================================================ +``` + +## Directory Structure + +``` +axiom-nmea/ +├── debug/ # Debug and analysis tools (see debug/README.md) +├── examples/ # Example applications +│ ├── quickstart/ # Minimal library usage example +│ ├── nmea-server-example/ # TCP NMEA sentence server +│ ├── pcap-to-nmea/ # Convert pcap to NMEA sentences +│ ├── sensor-monitor/ # Real-time sensor update monitor +│ ├── victron-bridge/ # Victron Venus OS integration +│ └── windy-station/ # Windy.com weather station +├── nmea-server/ # Dockerized NMEA server +├── raymarine_nmea/ # Core library +├── samples/ # Sample pcap files (not committed) +└── PROTOCOL.md # Protocol documentation +``` + +## Tools Included + +All debug tools are in the `debug/` directory. See [debug/README.md](debug/README.md) for full documentation. + +### Main Decoders +| Tool | Description | +|------|-------------| +| `debug/protobuf_decoder.py` | **Primary decoder** - all fields via proper protobuf parsing | +| `debug/raymarine_decoder.py` | Alternative decoder with live dashboard display | + +### Discovery Tools +| Tool | Description | +|------|-------------| +| `debug/battery_debug.py` | Deep nesting parser for battery fields (Field 14.3.4) | +| `debug/battery_finder.py` | Scans multicast groups for voltage-like values | +| `debug/tank_debug.py` | Raw Field 16 entry inspection | +| `debug/tank_finder.py` | Searches for tank level percentages | +| `debug/field_debugger.py` | Deep analysis of packet fields | + +### Analysis Tools +| Tool | Description | +|------|-------------| +| `debug/analyze_structure.py` | Packet structure analysis | +| `debug/field_mapping.py` | Documents the protobuf structure | +| `debug/protobuf_parser.py` | Lower-level wire format decoder | +| `debug/watch_field.py` | Monitor specific field values over time | + +### Wind/Heading Finders +| Tool | Description | +|------|-------------| +| `debug/wind_finder.py` | Searches for wind speed values | +| `debug/find_twd.py` | Searches for true wind direction | +| `debug/find_heading_vs_twd.py` | Compares heading and TWD values | +| `debug/find_consistent_heading.py` | Identifies stable heading fields | + +## Network Configuration + +### VLAN Setup + +Your network needs access to the Raymarine VLAN to receive multicast traffic: + +```bash +# Check VLAN interface exists +ip link show vlan.200 + +# If not, create it (requires VLAN support) +sudo ip link add link eth0 name vlan.200 type vlan id 200 +sudo ip link set dev vlan.200 up +sudo dhclient vlan.200 +``` + +### Testing Multicast Reception + +Before running the decoder, verify you can receive the multicast traffic: + +```bash +# Check if multicast traffic is arriving +tcpdump -i vlan.200 -c 10 'udp and dst net 224.0.0.0/4' + +# Look for specific ports +tcpdump -i vlan.200 -c 20 'udp port 2561 or udp port 2562 or udp port 2565' +``` + +## Sample PCAP Files + +Place your packet captures in the `samples/` directory for offline analysis: +- `samples/raymarine_sample.pcap` - General sample data +- `samples/raymarine_sample_TWD_62-70_HDG_29-35.pcap` - Known heading/wind angles +- `samples/raymarine_sample_twd_69-73.pcap` - Additional wind samples + +See [samples/README.md](samples/README.md) for capture instructions. Note: `.pcap` files are not committed to git. + +## License + +MIT License - Use freely for debugging your marine electronics. diff --git a/axiom-nmea/debug/README.md b/axiom-nmea/debug/README.md new file mode 100644 index 0000000..f1e8609 --- /dev/null +++ b/axiom-nmea/debug/README.md @@ -0,0 +1,155 @@ +# Debug Scripts + +This directory contains debugging and analysis tools for reverse-engineering the Raymarine LightHouse network protocol. These scripts are used to discover field mappings, locate sensor data, and understand the protobuf structure. + +## Protocol Analysis + +### `protobuf_decoder.py` +Full protobuf decoder with documented field mappings. Parses the nested protobuf structure and extracts sensor data (GPS, wind, depth, tanks, batteries, temperature). + +```bash +python protobuf_decoder.py -i 198.18.5.5 +python protobuf_decoder.py --pcap capture.pcap +``` + +### `protobuf_parser.py` +Low-level protobuf wire format parser without schema. Decodes the nested message structure to understand the protocol. + +### `raymarine_decoder.py` +Standalone Raymarine decoder that extracts sensor data from multicast packets. Outputs human-readable or JSON format. + +```bash +python raymarine_decoder.py -i 198.18.5.5 +python raymarine_decoder.py -i 198.18.5.5 --json +python raymarine_decoder.py --pcap raymarine_sample.pcap +``` + +### `packet_debug.py` +Dumps raw protobuf field structure from multicast packets. Shows all top-level fields, nested structures, and decoded values. + +```bash +python packet_debug.py -i 198.18.5.5 +``` + +### `field_debugger.py` +Interactive field mapper that displays all protobuf fields in a columnar format. Useful for correlating field values with real-world sensor readings. + +```bash +python field_debugger.py -i 192.168.1.100 # Live capture +python field_debugger.py --pcap capture.pcap # From file +python field_debugger.py --pcap capture.pcap -n 5 # Show 5 snapshots +``` + +### `field_mapping.py` +Documents the discovered field structure and validates against captured data. Shows the relationship between protobuf fields and sensor values. + +### `analyze_structure.py` +Analyzes packet header structure and protobuf nesting patterns. Groups packets by size and examines common headers. + +## Sensor Finders + +These scripts search for specific sensor values within the protobuf stream. + +### `find_cog_sog.py` +Searches all protobuf fields for values matching expected COG (Course Over Ground) and SOG (Speed Over Ground) ranges. + +```bash +python find_cog_sog.py -i 198.18.5.5 --cog-min 0 --cog-max 359 --sog-min 0 --sog-max 0.5 +python find_cog_sog.py -i 198.18.5.5 --show-all # Show ALL numeric fields +python find_cog_sog.py -i 198.18.5.5 -f 2 # Filter to field 2 only +``` + +### `wind_finder.py` +Searches for wind speed and direction values in captured packets. Expects wind speed in m/s (7-12) and direction in radians (1.0-1.7). + +### `find_twd.py` +Searches for True Wind Direction values in a specific degree range (e.g., 69-73 degrees). + +### `find_twd_precise.py` +Precise TWD finder with tighter tolerance for exact value matching. + +### `find_twd_hdg.py` +Finds both TWD and Heading offsets using known reference values. + +### `find_heading_vs_twd.py` +Correlates heading and TWD candidates. At anchor pointing into wind, heading and TWD should be within ~50 degrees. + +### `find_consistent_heading.py` +Finds offsets that show consistent heading-like values across multiple pcap files. + +### `pressure_finder.py` +Locates barometric pressure data by searching for values matching a known pressure reading. + +```bash +python pressure_finder.py -i YOUR_INTERFACE_IP -p 1021 # Known pressure in mbar +``` + +### `battery_finder.py` +Scans all multicast groups for values matching expected battery voltages (12V, 24V systems). + +```bash +python battery_finder.py -i 198.18.5.5 -t 10 +python battery_finder.py -i 198.18.5.5 -v # Verbose mode +``` + +### `tank_finder.py` +Scans multicast groups for values matching expected tank level percentages. + +## Field-Specific Debug Tools + +### `battery_debug.py` +Dumps raw protobuf entries to find battery data fields. Supports deep nesting analysis (e.g., Field 14.3.4 for engine battery). + +```bash +python battery_debug.py -i 198.18.5.5 -t 5 +python battery_debug.py -i 198.18.5.5 -f 14 # Focus on Field 14 +``` + +### `tank_debug.py` +Dumps raw Field 16 entries to discover tank IDs and status values. + +### `debug_wind.py` +Examines actual packet bytes at known wind data offsets. + +### `debug_decode.py` +Simulates the wind extraction logic and shows packet size distribution. + +### `debug_field13.py` +Debugs Field 13 (Wind/Navigation) extraction across different packet sizes. + +### `watch_field.py` +Monitors a specific protobuf field path across incoming packets. + +```bash +python watch_field.py -i 192.168.1.100 --field 7.1 # Watch depth +python watch_field.py --pcap capture.pcap --field 13.4 # Watch TWD +``` + +## Offset Comparison Tools + +### `compare_offsets.py` +Compares old fixed-byte offsets vs protobuf field-based offsets for wind direction. + +### `compare_heading_both_pcaps.py` +Compares heading and TWD candidates between two different pcap files to validate consistency. + +### `check_006b.py` +Thoroughly checks offset 0x006b across all packets of various sizes. + +## Usage Notes + +Most scripts require either: +- `-i, --interface` - The IP address of the interface connected to the Raymarine network +- `--pcap` - Path to a captured pcap file for offline analysis + +Common multicast groups: +- `226.192.206.102:2565` - Primary sensor data +- `226.192.206.98:2561` - Navigation sensors +- `239.2.1.1:2154` - Additional sensor data (tanks, engines) + +## Data Formats + +- Angles are stored in **radians** (multiply by 57.2958 for degrees) +- Speeds are stored in **m/s** (multiply by 1.94384 for knots) +- Depth is stored in **meters** +- Temperature is stored in **Kelvin** (subtract 273.15 for Celsius) diff --git a/axiom-nmea/debug/analyze_structure.py b/axiom-nmea/debug/analyze_structure.py new file mode 100644 index 0000000..b0a7667 --- /dev/null +++ b/axiom-nmea/debug/analyze_structure.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +""" +Analyze packet structure to understand why offsets vary. +Look for common headers and protobuf nesting patterns. +""" + +import struct + +def read_pcap(filename): + packets = [] + with open(filename, 'rb') as f: + header = f.read(24) + magic = struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + +packets = read_pcap("raymarine_sample_TWD_62-70_HDG_29-35.pcap") + +# Group by size +by_size = {} +for pkt in packets: + pkt_len = len(pkt) + if pkt_len not in by_size: + by_size[pkt_len] = [] + by_size[pkt_len].append(pkt) + +print("=" * 70) +print("PACKET HEADER ANALYSIS") +print("=" * 70) + +for pkt_len in sorted(by_size.keys()): + if pkt_len < 100: + continue + + pkts = by_size[pkt_len][:3] # First 3 packets of each size + + print(f"\n{'='*70}") + print(f"PACKET SIZE: {pkt_len} bytes ({len(by_size[pkt_len])} total)") + print("=" * 70) + + pkt = pkts[0] + + # Show first 128 bytes as hex + print("\nFirst 128 bytes (hex):") + for row in range(0, min(128, pkt_len), 16): + hex_str = ' '.join(f'{b:02x}' for b in pkt[row:row+16]) + ascii_str = ''.join(chr(b) if 32 <= b < 127 else '.' for b in pkt[row:row+16]) + print(f" 0x{row:04x}: {hex_str:<48} {ascii_str}") + + # Look for the GPS pattern (0x09 followed by lat, 0x11 followed by lon) + gps_offset = None + for i in range(0x20, min(pkt_len - 18, 0x60)): + if pkt[i] == 0x09 and i + 9 < pkt_len and pkt[i + 9] == 0x11: + lat = struct.unpack(' 1: + gps_offset = i + print(f"\n GPS found at offset 0x{i:04x}: {lat:.6f}, {lon:.6f}") + break + + # Look for string patterns (device name) + for i in range(0x10, min(pkt_len - 10, 0x40)): + if pkt[i:i+5] == b'AXIOM': + print(f" 'AXIOM' string at offset 0x{i:04x}") + # Show surrounding context + end = min(i + 30, pkt_len) + print(f" Context: {pkt[i:end]}") + break + + # Analyze protobuf wire types at key offsets + print(f"\n Protobuf tags at key offsets:") + for offset in [0x0070, 0x00a0, 0x00a7, 0x00c5, 0x00fc]: + if offset < pkt_len: + tag = pkt[offset] + wire_type = tag & 0x07 + field_num = tag >> 3 + wire_names = {0: 'varint', 1: 'fixed64', 2: 'length', 5: 'fixed32'} + print(f" 0x{offset:04x}: tag=0x{tag:02x} (field {field_num}, {wire_names.get(wire_type, '?')})") diff --git a/axiom-nmea/debug/battery_debug.py b/axiom-nmea/debug/battery_debug.py new file mode 100644 index 0000000..ea05b9d --- /dev/null +++ b/axiom-nmea/debug/battery_debug.py @@ -0,0 +1,453 @@ +#!/usr/bin/env python3 +""" +Battery Debug - Dump raw protobuf entries to find battery data fields. + +Scans for fields that might contain battery data (voltage, current, SoC). +Supports deep nesting (e.g., Field 14.3.4 for engine battery). +""" + +import struct +import socket +import time +import threading +from typing import Dict, List, Any, Optional + +WIRE_VARINT = 0 +WIRE_FIXED64 = 1 +WIRE_LENGTH = 2 +WIRE_FIXED32 = 5 + +HEADER_SIZE = 20 + +# Fields to specifically look for battery data +# Field 14 = Engine data (contains battery at 14.3.4) +# Field 20 = House batteries +BATTERY_CANDIDATE_FIELDS = {14, 17, 18, 19, 20, 21, 22, 23, 24, 25} + +MULTICAST_GROUPS = [ + ("226.192.206.102", 2565), # Main sensor data + ("239.2.1.1", 2154), # May contain additional sensor data +] + + +class ProtobufParser: + def __init__(self, data: bytes): + self.data = data + self.pos = 0 + + def remaining(self): + return len(self.data) - self.pos + + def read_varint(self) -> int: + result = 0 + shift = 0 + while self.pos < len(self.data): + byte = self.data[self.pos] + self.pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + return result + + def read_fixed32(self) -> bytes: + val = self.data[self.pos:self.pos + 4] + self.pos += 4 + return val + + def read_fixed64(self) -> bytes: + val = self.data[self.pos:self.pos + 8] + self.pos += 8 + return val + + def read_length_delimited(self) -> bytes: + length = self.read_varint() + val = self.data[self.pos:self.pos + length] + self.pos += length + return val + + def parse_all_fields(self) -> Dict[int, List[Any]]: + """Parse and collect all fields, grouping repeated fields.""" + fields = {} + + while self.pos < len(self.data): + if self.remaining() < 1: + break + try: + start_pos = self.pos + tag = self.read_varint() + field_num = tag >> 3 + wire_type = tag & 0x07 + + if field_num == 0 or field_num > 1000: + break + + if wire_type == WIRE_VARINT: + value = ('varint', self.read_varint()) + elif wire_type == WIRE_FIXED64: + raw = self.read_fixed64() + try: + d = struct.unpack(' List[tuple]: + """Recursively parse nested protobuf and return list of (path, type, value) tuples.""" + results = [] + pos = 0 + + if depth > max_depth: + return results + + while pos < len(data): + if pos >= len(data): + break + try: + # Read tag + tag_byte = data[pos] + pos += 1 + + # Handle multi-byte varints for tag + tag = tag_byte & 0x7F + shift = 7 + while tag_byte & 0x80 and pos < len(data): + tag_byte = data[pos] + pos += 1 + tag |= (tag_byte & 0x7F) << shift + shift += 7 + + field_num = tag >> 3 + wire_type = tag & 0x07 + + if field_num == 0 or field_num > 100: + break + + field_path = f"{path}.{field_num}" if path else str(field_num) + + if wire_type == WIRE_VARINT: + # Read varint value + val = 0 + shift = 0 + while pos < len(data): + byte = data[pos] + pos += 1 + val |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + results.append((field_path, 'varint', val)) + + elif wire_type == WIRE_FIXED32: + raw = data[pos:pos + 4] + pos += 4 + try: + f = struct.unpack('= 2: + nested_results = self.parse_nested_deep(raw, field_path, depth + 1, max_depth) + if nested_results: + results.extend(nested_results) + else: + # Couldn't parse as nested, store as bytes + results.append((field_path, 'bytes', raw.hex()[:40])) + else: + results.append((field_path, 'bytes', raw.hex())) + + else: + break + + except Exception as e: + break + + return results + + def parse_nested_entry(self, data: bytes) -> dict: + """Parse a nested protobuf entry and return all fields (single level).""" + entry = {'fields': {}} + pos = 0 + + while pos < len(data): + if pos >= len(data): + break + try: + # Read tag + tag_byte = data[pos] + pos += 1 + + # Handle multi-byte varints for tag + tag = tag_byte & 0x7F + shift = 7 + while tag_byte & 0x80 and pos < len(data): + tag_byte = data[pos] + pos += 1 + tag |= (tag_byte & 0x7F) << shift + shift += 7 + + field_num = tag >> 3 + wire_type = tag & 0x07 + + if field_num == 0 or field_num > 100: + break + + if wire_type == WIRE_VARINT: + # Read varint value + val = 0 + shift = 0 + while pos < len(data): + byte = data[pos] + pos += 1 + val |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + entry['fields'][field_num] = ('varint', val) + + elif wire_type == WIRE_FIXED32: + raw = data[pos:pos + 4] + pos += 4 + try: + f = struct.unpack(' Optional[str]: + """Check if a float value looks like a battery voltage.""" + if 10.0 <= val <= 16.0: + return "12V system" + if 20.0 <= val <= 32.0: + return "24V system" + if 40.0 <= val <= 60.0: + return "48V system" + return None + + +def print_nested_field(raw_data: bytes, field_num: int, indent: str = " "): + """Print a nested field with deep parsing.""" + parser = ProtobufParser(raw_data) + deep_results = parser.parse_nested_deep(raw_data, str(field_num)) + + # Group results by depth for better display + print(f"{indent}Deep parse of Field {field_num} (length={len(raw_data)}):") + + for path, vtype, value in deep_results: + depth = path.count('.') + sub_indent = indent + " " * depth + + if vtype == 'float': + voltage_hint = is_voltage_like(value) + if voltage_hint: + print(f"{sub_indent}Field {path}: {value:.2f} ({vtype}) <- VOLTAGE? ({voltage_hint})") + elif value != value: # NaN + print(f"{sub_indent}Field {path}: nan ({vtype})") + else: + print(f"{sub_indent}Field {path}: {value:.4f} ({vtype})") + elif vtype == 'double': + voltage_hint = is_voltage_like(value) + if voltage_hint: + print(f"{sub_indent}Field {path}: {value:.2f} ({vtype}) <- VOLTAGE? ({voltage_hint})") + else: + print(f"{sub_indent}Field {path}: {value:.4f} ({vtype})") + elif vtype == 'varint': + print(f"{sub_indent}Field {path}: {value} ({vtype})") + else: + print(f"{sub_indent}Field {path}: {value} ({vtype})") + + +def scan_packet(data: bytes, group: str, port: int, target_field: Optional[int] = None): + """Scan a packet and dump potential battery-related fields.""" + if len(data) < HEADER_SIZE + 5: + return + + proto_data = data[HEADER_SIZE:] + parser = ProtobufParser(proto_data) + all_fields = parser.parse_all_fields() + + print(f"\n{'='*70}") + print(f"Packet from {group}:{port} (size: {len(data)} bytes)") + print(f"Top-level fields present: {sorted(all_fields.keys())}") + print(f"{'='*70}") + + # If a specific field is targeted, only show that one + fields_to_check = {target_field} if target_field else BATTERY_CANDIDATE_FIELDS + + # Check candidate fields for battery data + for field_num in sorted(fields_to_check): + if field_num in all_fields: + entries = all_fields[field_num] + print(f"\n Field {field_num}: {len(entries)} entries") + + for i, entry in enumerate(entries): + if entry[0] == 'length': + raw_data = entry[1] + print(f"\n Entry {i+1}:") + print_nested_field(raw_data, field_num, " ") + + elif entry[0] in ('float', 'double'): + voltage_hint = is_voltage_like(entry[1]) + if voltage_hint: + print(f" Value: {entry[1]:.2f} <- VOLTAGE? ({voltage_hint})") + else: + print(f" Value: {entry[1]:.4f}") + else: + print(f" Value: {entry}") + + # Deep scan ALL fields for voltage-like values + print(f"\n Deep scanning all fields for voltage-like values:") + found_any = False + for field_num, entries in sorted(all_fields.items()): + for entry in entries: + if entry[0] == 'length': + raw_data = entry[1] + deep_parser = ProtobufParser(raw_data) + deep_results = deep_parser.parse_nested_deep(raw_data, str(field_num)) + for path, vtype, value in deep_results: + if vtype in ('float', 'double') and is_voltage_like(value): + print(f" Field {path}: {value:.2f}V ({is_voltage_like(value)})") + found_any = True + elif entry[0] == 'float' and is_voltage_like(entry[1]): + print(f" Field {field_num}: {entry[1]:.2f}V ({is_voltage_like(entry[1])})") + found_any = True + elif entry[0] == 'double' and is_voltage_like(entry[1]): + print(f" Field {field_num}: {entry[1]:.2f}V ({is_voltage_like(entry[1])})") + found_any = True + + if not found_any: + print(f" (no voltage-like values found)") + + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Debug potential battery data fields") + parser.add_argument('-i', '--interface', required=True, help='Interface IP') + parser.add_argument('-t', '--time', type=int, default=5, help='Capture time (seconds)') + parser.add_argument('-g', '--group', default="226.192.206.102", help='Multicast group') + parser.add_argument('-p', '--port', type=int, default=2565, help='Port') + parser.add_argument('-f', '--field', type=int, help='Focus on specific field number (e.g., 14)') + args = parser.parse_args() + + print(f"Scanning for battery data fields...") + if args.field: + print(f"Focusing on Field {args.field} with deep nesting") + else: + print(f"Looking at fields: {sorted(BATTERY_CANDIDATE_FIELDS)}") + print(f"Target voltages: Aft House ~26.3V, Stern House ~27.2V, Port Engine ~26.5V") + print(f"\nCapturing from {args.group}:{args.port} for {args.time} seconds...") + + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, 'SO_REUSEPORT'): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind(('', args.port)) + mreq = struct.pack("4s4s", socket.inet_aton(args.group), socket.inet_aton(args.interface)) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + sock.settimeout(1.0) + + seen_sizes = set() + end_time = time.time() + args.time + + try: + while time.time() < end_time: + try: + data, _ = sock.recvfrom(65535) + # Only process each unique packet size once + if len(data) not in seen_sizes: + seen_sizes.add(len(data)) + scan_packet(data, args.group, args.port, args.field) + except socket.timeout: + continue + except KeyboardInterrupt: + pass + finally: + sock.close() + + print("\n\nDone.") + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/debug/battery_finder.py b/axiom-nmea/debug/battery_finder.py new file mode 100644 index 0000000..40a40f0 --- /dev/null +++ b/axiom-nmea/debug/battery_finder.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +""" +Battery Finder - Scan all multicast groups for values matching expected battery voltages. + +Searches for house battery voltages (26.3V and 27.2V) across all protobuf fields. +""" + +import struct +import socket +import time +import threading +from typing import Dict, Any, Optional, List, Tuple + +WIRE_VARINT = 0 +WIRE_FIXED64 = 1 +WIRE_LENGTH = 2 +WIRE_FIXED32 = 5 + +HEADER_SIZE = 20 + +MULTICAST_GROUPS = [ + ("226.192.206.98", 2561), + ("226.192.206.99", 2562), + ("226.192.206.100", 2563), + ("226.192.206.101", 2564), + ("226.192.206.102", 2565), + ("226.192.219.0", 3221), + ("239.2.1.1", 2154), +] + +# Target voltage values to find (with tolerance) +# Aft House: 26.3V, Stern House: 27.2V +TARGET_VOLTAGES = [ + (26.0, 26.6), # Aft house battery ~26.3V + (26.9, 27.5), # Stern house battery ~27.2V + (12.0, 15.0), # 12V battery range (in case they're 12V systems) + (24.0, 29.0), # General 24V battery range +] + + +class ProtobufParser: + def __init__(self, data: bytes): + self.data = data + self.pos = 0 + + def read_varint(self) -> int: + result = 0 + shift = 0 + while self.pos < len(self.data): + byte = self.data[self.pos] + self.pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + return result + + def parse(self, path: str = "") -> List[Tuple[str, str, Any]]: + """Parse and return list of (path, type, value) for all fields.""" + results = [] + while self.pos < len(self.data): + try: + tag = self.read_varint() + field_num = tag >> 3 + wire_type = tag & 0x07 + + if field_num == 0 or field_num > 1000: + break + + field_path = f"{path}.{field_num}" if path else str(field_num) + + if wire_type == WIRE_VARINT: + value = self.read_varint() + results.append((field_path, "varint", value)) + elif wire_type == WIRE_FIXED64: + raw = self.data[self.pos:self.pos + 8] + self.pos += 8 + try: + d = struct.unpack(' Optional[str]: + """Check if value matches expected battery voltage ranges.""" + if 26.0 <= val <= 26.6: + return "Aft House (~26.3V)" + if 26.9 <= val <= 27.5: + return "Stern House (~27.2V)" + if 12.0 <= val <= 15.0: + return "12V battery range" + if 24.0 <= val <= 29.0: + return "24V battery range" + return None + + +def scan_packet(data: bytes, group: str, port: int, verbose: bool = False): + """Scan a packet for voltage values.""" + if len(data) < HEADER_SIZE + 5: + return + + proto_data = data[HEADER_SIZE:] + parser = ProtobufParser(proto_data) + fields = parser.parse() + + matches = [] + for path, vtype, value in fields: + if isinstance(value, (int, float)): + match_desc = is_voltage_value(value) + if match_desc: + matches.append((path, vtype, value, match_desc)) + + if matches: + print(f"\n{'='*70}") + print(f"VOLTAGE MATCH on {group}:{port} (packet size: {len(data)})") + print(f"{'='*70}") + for path, vtype, value, desc in matches: + print(f" Field {path} ({vtype}): {value:.2f}V <- {desc}") + + # Show context: look at parent field structure + if verbose: + print(f"\nAll numeric values in packet:") + for path, vtype, value in fields: + if isinstance(value, (int, float)) and value != 0: + print(f" {path}: {value} ({vtype})") + + +class MulticastScanner: + def __init__(self, interface_ip: str, verbose: bool = False): + self.interface_ip = interface_ip + self.verbose = verbose + self.running = False + self.lock = threading.Lock() + + def _create_socket(self, group: str, port: int): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, 'SO_REUSEPORT'): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind(('', port)) + mreq = struct.pack("4s4s", socket.inet_aton(group), socket.inet_aton(self.interface_ip)) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + sock.settimeout(1.0) + return sock + + def _listen(self, sock, group: str, port: int): + seen_sizes = set() + while self.running: + try: + data, _ = sock.recvfrom(65535) + # Only process each unique packet size once per group + size_key = len(data) + if size_key not in seen_sizes: + seen_sizes.add(size_key) + with self.lock: + scan_packet(data, group, port, self.verbose) + except socket.timeout: + continue + except: + pass + + def start(self): + self.running = True + threads = [] + for group, port in MULTICAST_GROUPS: + try: + sock = self._create_socket(group, port) + t = threading.Thread(target=self._listen, args=(sock, group, port), daemon=True) + t.start() + threads.append(t) + print(f"Scanning {group}:{port}") + except Exception as e: + print(f"Error on {group}:{port}: {e}") + return threads + + def stop(self): + self.running = False + + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Find battery voltage values in multicast data") + parser.add_argument('-i', '--interface', required=True, help='Interface IP') + parser.add_argument('-t', '--time', type=int, default=10, help='Scan duration (seconds)') + parser.add_argument('-v', '--verbose', action='store_true', help='Show all numeric values') + args = parser.parse_args() + + print(f"Scanning for battery voltages:") + print(f" - Aft House: ~26.3V (range 26.0-26.6)") + print(f" - Stern House: ~27.2V (range 26.9-27.5)") + print(f" - Also checking 12V and 24V ranges") + print(f"\nWill scan for {args.time} seconds\n") + + scanner = MulticastScanner(args.interface, args.verbose) + scanner.start() + + try: + time.sleep(args.time) + except KeyboardInterrupt: + pass + finally: + scanner.stop() + print("\nDone scanning") + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/debug/check_006b.py b/axiom-nmea/debug/check_006b.py new file mode 100644 index 0000000..71d2b9a --- /dev/null +++ b/axiom-nmea/debug/check_006b.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Check offset 0x006b thoroughly across all packets.""" + +import struct +from collections import Counter + +def decode_float(data, offset): + if offset + 4 > len(data): + return None + try: + val = struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + +OFFSET = 0x006b + +for pcap in ["raymarine_sample_twd_69-73.pcap", "raymarine_sample.pcap"]: + print(f"\n{'='*60}") + print(f"FILE: {pcap}") + print(f"OFFSET: 0x{OFFSET:04x}") + print("="*60) + + packets = read_pcap(pcap) + + # Group by packet size + by_size = {} + for pkt in packets: + pkt_len = len(pkt) + if pkt_len not in by_size: + by_size[pkt_len] = [] + by_size[pkt_len].append(pkt) + + for pkt_len in sorted(by_size.keys()): + if pkt_len < 120: # Skip small packets + continue + + pkts = by_size[pkt_len] + vals = [] + for pkt in pkts: + v = decode_float(pkt, OFFSET) + if v is not None: + vals.append(v) + + if vals: + # Convert to degrees + degs = [v * 57.2958 for v in vals if 0 <= v <= 6.5] + if degs: + avg = sum(degs) / len(degs) + print(f" {pkt_len:5d} bytes ({len(pkts):3d} pkts): avg={avg:6.1f}°, range={min(degs):.1f}°-{max(degs):.1f}°") + else: + # Not radians - show raw + print(f" {pkt_len:5d} bytes ({len(pkts):3d} pkts): raw values (not radians)") diff --git a/axiom-nmea/debug/compare_heading_both_pcaps.py b/axiom-nmea/debug/compare_heading_both_pcaps.py new file mode 100644 index 0000000..fe26897 --- /dev/null +++ b/axiom-nmea/debug/compare_heading_both_pcaps.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Compare heading candidates between both pcap files.""" + +import struct + +def decode_float(data, offset): + if offset + 4 > len(data): + return None + try: + val = struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + +# Heading candidate offsets by packet size (from TWD analysis) +HEADING_OFFSETS = { + 344: 0x0144, + 446: 0x0189, + 788: 0x0081, + 888: 0x0081, + 931: 0x0081, + 1031: 0x0081, + 1472: 0x0088, +} + +# TWD offset (consistent across sizes) +TWD_OFFSET = 0x006b + +print("=" * 70) +print("COMPARING TWD (0x006b) AND HEADING CANDIDATES ACROSS BOTH PCAPS") +print("=" * 70) + +for pcap_file in ["raymarine_sample.pcap", "raymarine_sample_twd_69-73.pcap"]: + print(f"\n{'='*70}") + print(f"FILE: {pcap_file}") + print("=" * 70) + + packets = read_pcap(pcap_file) + + # Group by packet size + by_size = {} + for pkt in packets: + pkt_len = len(pkt) + if pkt_len not in by_size: + by_size[pkt_len] = [] + by_size[pkt_len].append(pkt) + + print(f"\n{'Pkt Size':<10} {'Count':<8} {'TWD (0x006b)':<18} {'Heading':<18} {'Diff':<10}") + print("-" * 70) + + for pkt_len in sorted(HEADING_OFFSETS.keys()): + if pkt_len not in by_size: + continue + + pkts = by_size[pkt_len] + heading_offset = HEADING_OFFSETS[pkt_len] + + # Get all values + twd_vals = [] + hdg_vals = [] + + for pkt in pkts: + twd = decode_float(pkt, TWD_OFFSET) + hdg = decode_float(pkt, heading_offset) + + if twd and 0 <= twd <= 6.5: + twd_vals.append(twd * 57.2958) + if hdg and 0 <= hdg <= 6.5: + hdg_vals.append(hdg * 57.2958) + + if twd_vals and hdg_vals: + twd_avg = sum(twd_vals) / len(twd_vals) + hdg_avg = sum(hdg_vals) / len(hdg_vals) + diff = abs(twd_avg - hdg_avg) + + twd_str = f"{twd_avg:.1f}° ({min(twd_vals):.0f}-{max(twd_vals):.0f})" + hdg_str = f"{hdg_avg:.1f}° ({min(hdg_vals):.0f}-{max(hdg_vals):.0f})" + + print(f"{pkt_len:<10} {len(pkts):<8} {twd_str:<18} {hdg_str:<18} {diff:.1f}°") + elif twd_vals: + twd_avg = sum(twd_vals) / len(twd_vals) + print(f"{pkt_len:<10} {len(pkts):<8} {twd_avg:.1f}° ---") + else: + print(f"{pkt_len:<10} {len(pkts):<8} --- ---") + +print("\n" + "=" * 70) +print("INTERPRETATION") +print("=" * 70) +print(""" +If heading and TWD are consistent across both captures: +- The offsets are correct for those data types +- Difference should be within ~50° (boat at anchor pointing into wind) + +Expected relationships: +- TWD capture: TWD ~69-73°, so heading should be ~20-120° +- Original capture: Need to check what values make sense +""") diff --git a/axiom-nmea/debug/compare_offsets.py b/axiom-nmea/debug/compare_offsets.py new file mode 100644 index 0000000..0745382 --- /dev/null +++ b/axiom-nmea/debug/compare_offsets.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Compare old offsets vs new 0x006b offset for wind direction.""" + +import struct +from collections import defaultdict + +def decode_float(data, offset): + if offset + 4 > len(data): + return None + try: + val = struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + +# Current offsets from decoder +OLD_OFFSETS = { + 344: 0x00a0, + 446: 0x00a7, + 788: 0x00c5, + 888: 0x00c5, + 931: 0x00c5, + 1031: 0x00c5, + 1472: 0x00fc, +} + +NEW_OFFSET = 0x006b + +for pcap_file in ["raymarine_sample.pcap", "raymarine_sample_twd_69-73.pcap"]: + print(f"\n{'='*70}") + print(f"FILE: {pcap_file}") + print("="*70) + + packets = read_pcap(pcap_file) + print(f"Loaded {len(packets)} packets\n") + + print(f"{'Pkt Size':<10} {'Old Offset':<12} {'Old Value':<15} {'New (0x006b)':<15}") + print("-" * 55) + + for pkt_len in sorted(OLD_OFFSETS.keys()): + old_offset = OLD_OFFSETS[pkt_len] + + # Find packets of this size + matching = [p for p in packets if len(p) == pkt_len] + if not matching: + continue + + # Sample first packet of this size + pkt = matching[0] + + old_val = decode_float(pkt, old_offset) + new_val = decode_float(pkt, NEW_OFFSET) + + old_deg = f"{old_val * 57.2958:.1f}°" if old_val and 0 <= old_val <= 6.5 else "invalid" + new_deg = f"{new_val * 57.2958:.1f}°" if new_val and 0 <= new_val <= 6.5 else "invalid" + + print(f"{pkt_len:<10} 0x{old_offset:04x} {old_deg:<15} {new_deg:<15}") diff --git a/axiom-nmea/debug/debug_decode.py b/axiom-nmea/debug/debug_decode.py new file mode 100644 index 0000000..2e419b4 --- /dev/null +++ b/axiom-nmea/debug/debug_decode.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Debug the full decode process.""" + +import struct +from collections import Counter + +def decode_float(data, offset): + if offset + 4 > len(data): + return None + try: + return struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + +packets = read_pcap("raymarine_sample.pcap") + +# Count packet sizes +size_counts = Counter(len(p) for p in packets) +print("Packet size distribution:") +for size, count in sorted(size_counts.items(), key=lambda x: -x[1])[:15]: + print(f" {size:5d} bytes: {count:4d} packets") + +print("\n" + "="*60) +print("Simulating wind extraction logic:") +print("="*60) + +last_speed = None +last_dir = None +extractions = 0 + +for pkt in packets: + pkt_len = len(pkt) + + # Same logic as decoder + if 340 <= pkt_len <= 350: + offset_pairs = [(0x00a5, 0x00a0), (0x00c3, 0x00a0), (0x00c8, 0x00a0)] + elif 440 <= pkt_len <= 500: + offset_pairs = [(0x00ac, 0x00a7), (0x00ca, 0x00a7), (0x00b1, 0x00a7)] + elif 780 <= pkt_len <= 1100: + offset_pairs = [(0x00ca, 0x00c5), (0x00e8, 0x0090), (0x00cf, 0x00c5)] + elif 1400 <= pkt_len <= 1500: + offset_pairs = [(0x0101, 0x00fc), (0x0106, 0x00fc), (0x011f, 0x00fc)] + else: + offset_pairs = [(0x00ca, 0x00c5), (0x00a5, 0x00a0)] + + for speed_offset, dir_offset in offset_pairs: + if speed_offset + 4 > pkt_len or dir_offset + 4 > pkt_len: + continue + + speed_val = decode_float(pkt, speed_offset) + dir_val = decode_float(pkt, dir_offset) + + if speed_val is None or dir_val is None: + continue + + # Validate + if not (0 < speed_val < 60): + continue + if not (0 <= dir_val <= 6.5): + continue + + speed_kts = speed_val * 1.94384 + dir_deg = (dir_val * 57.2958) % 360 + + last_speed = speed_kts + last_dir = dir_deg + extractions += 1 + break + +print(f"\nTotal successful extractions: {extractions}") +print(f"Last wind speed: {last_speed:.1f} kts" if last_speed else "No speed") +print(f"Last wind direction: {last_dir:.1f}°" if last_dir else "No direction") diff --git a/axiom-nmea/debug/debug_field13.py b/axiom-nmea/debug/debug_field13.py new file mode 100644 index 0000000..b350138 --- /dev/null +++ b/axiom-nmea/debug/debug_field13.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Debug Field 13 extraction across different packet sizes.""" + +import struct +from protobuf_decoder import ProtobufParser, HEADER_SIZE, WIRE_FIXED32 + +def decode_float(raw): + if len(raw) != 4: + return None + try: + val = struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + +packets = read_pcap("raymarine_sample_TWD_62-70_HDG_29-35.pcap") + +# Group by size +by_size = {} +for pkt in packets: + size = len(pkt) + if size not in by_size: + by_size[size] = [] + by_size[size].append(pkt) + +print("Analyzing Field 13 by packet size:") +print("Expected: TWD 62-70°, HDG 29-35°") +print("=" * 70) + +for size in sorted(by_size.keys()): + if size < 200: + continue + + pkts = by_size[size][:3] # First 3 of each size + + print(f"\n{size} bytes ({len(by_size[size])} packets):") + + for pkt in pkts: + proto_data = pkt[HEADER_SIZE:] + parser = ProtobufParser(proto_data) + fields = parser.parse_message() + + if 13 not in fields: + print(" No Field 13") + continue + + f13 = fields[13] + if not f13.children: + print(" Field 13 has no children") + continue + + # Show all float fields in Field 13 + floats = [] + for fnum, child in sorted(f13.children.items()): + if child.wire_type == WIRE_FIXED32: + val = decode_float(child.value) + if val is not None: + deg = val * 57.2958 + floats.append(f"f{fnum}={deg:.1f}°") + + print(f" {', '.join(floats)}") diff --git a/axiom-nmea/debug/debug_wind.py b/axiom-nmea/debug/debug_wind.py new file mode 100644 index 0000000..872adc8 --- /dev/null +++ b/axiom-nmea/debug/debug_wind.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Debug wind extraction by examining actual packet bytes.""" + +import struct + +PCAP_MAGIC = 0xa1b2c3d4 + +def decode_float(data, offset): + if offset + 4 > len(data): + return None + try: + return struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + +packets = read_pcap("raymarine_sample.pcap") + +# Find packets of different sizes +sizes_seen = {} +for pkt in packets: + plen = len(pkt) + if plen not in sizes_seen: + sizes_seen[plen] = pkt + +print("Examining wind data at known offsets:\n") + +for target_size in [344, 446, 788, 888, 1472]: + # Find a packet close to this size + for plen, pkt in sizes_seen.items(): + if abs(plen - target_size) <= 10: + print(f"=== Packet size {plen} ===") + + # Show bytes around expected wind offsets + if 340 <= plen <= 350: + offsets = [0x00a0, 0x00a5, 0x00c3, 0x00c8] + elif 440 <= plen <= 500: + offsets = [0x00a7, 0x00ac, 0x00ca, 0x00b1] + elif 780 <= plen <= 900: + offsets = [0x0090, 0x00c5, 0x00ca, 0x00cf] + elif 1400 <= plen <= 1500: + offsets = [0x00fc, 0x0101, 0x0106] + else: + offsets = [] + + for off in offsets: + if off + 8 <= plen: + hex_bytes = ' '.join(f'{b:02x}' for b in pkt[off:off+8]) + float_val = decode_float(pkt, off) + # Also try treating as m/s and rad + speed_kts = float_val * 1.94384 if float_val else 0 + dir_deg = float_val * 57.2958 if float_val else 0 + print(f" 0x{off:04x}: {hex_bytes} -> float={float_val:.4f} ({speed_kts:.1f} kts or {dir_deg:.1f}°)") + print() + break diff --git a/axiom-nmea/debug/dock_finder.py b/axiom-nmea/debug/dock_finder.py new file mode 100755 index 0000000..1b34847 --- /dev/null +++ b/axiom-nmea/debug/dock_finder.py @@ -0,0 +1,561 @@ +#!/usr/bin/env python3 +""" +Dock Finder - Find SOG and COG fields while stationary at dock. + +When at dock: +- SOG bounces between 0.0 and 0.2 kts (0 to ~0.1 m/s) +- COG jumps wildly between 0 and 359 degrees (0 to ~6.28 radians) + +This script looks for paired fields that show these patterns: +1. Speed: small positive values near zero (0-0.1 m/s → 0-0.2 kts) +2. Angle: values spanning nearly the full 0-2π range (radians) + +The script tracks variance over time to identify fluctuating fields. + +Usage: + python dock_finder.py -i 198.18.5.5 + python dock_finder.py -i 198.18.5.5 --samples 20 --interval 0.5 +""" + +import argparse +import math +import os +import signal +import socket +import struct +import sys +import time +from collections import defaultdict +from copy import copy +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Dict, List, Optional, Set, Tuple + +# Add parent directory to path for library import +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from raymarine_nmea.protocol.parser import ProtobufParser, ProtoField +from raymarine_nmea.protocol.constants import ( + WIRE_VARINT, WIRE_FIXED64, WIRE_LENGTH, WIRE_FIXED32, + HEADER_SIZE, RAD_TO_DEG, MS_TO_KTS, +) +from raymarine_nmea.sensors import MULTICAST_GROUPS + +running = True + + +def signal_handler(signum, frame): + global running + running = False + + +@dataclass +class FieldStats: + """Statistics for a field across multiple samples.""" + path: str + wire_type: str + values: List[float] + + @property + def count(self) -> int: + return len(self.values) + + @property + def min_val(self) -> float: + return min(self.values) if self.values else 0 + + @property + def max_val(self) -> float: + return max(self.values) if self.values else 0 + + @property + def range_val(self) -> float: + return self.max_val - self.min_val + + @property + def mean(self) -> float: + return sum(self.values) / len(self.values) if self.values else 0 + + @property + def variance(self) -> float: + if len(self.values) < 2: + return 0 + mean = self.mean + return sum((v - mean) ** 2 for v in self.values) / len(self.values) + + @property + def std_dev(self) -> float: + return math.sqrt(self.variance) + + def is_sog_candidate(self) -> bool: + """Check if this could be SOG at dock (0-0.1 m/s, some variance).""" + # Must be small positive values in m/s range + if self.min_val < -0.01: # Allow tiny negative noise + return False + if self.max_val > 0.2: # Max 0.2 m/s ≈ 0.4 kts + return False + if self.max_val < 0.001: # Must have some value + return False + # Should have some variance (dock bouncing) + if self.range_val < 0.001: + return False + return True + + def is_cog_candidate(self) -> bool: + """Check if this could be COG at dock (full circle jumps in radians).""" + # Must be in valid radian range (0 to 2π ≈ 6.28) + if self.min_val < -0.1: + return False + if self.max_val > 7.0: # Allow slightly over 2π + return False + # At dock, COG jumps wildly - expect large range + # Range should span significant portion of circle (at least 90°) + min_range_rad = math.pi / 2 # 90 degrees in radians + if self.range_val < min_range_rad: + return False + # Variance should be high + if self.std_dev < 0.5: # Radians + return False + return True + + def as_degrees(self) -> Tuple[float, float, float]: + """Return min, max, mean as degrees.""" + return ( + (self.min_val * RAD_TO_DEG) % 360, + (self.max_val * RAD_TO_DEG) % 360, + (self.mean * RAD_TO_DEG) % 360 + ) + + def as_knots(self) -> Tuple[float, float, float]: + """Return min, max, mean as knots.""" + return ( + self.min_val * MS_TO_KTS, + self.max_val * MS_TO_KTS, + self.mean * MS_TO_KTS + ) + + +def decode_float(raw: bytes) -> Optional[float]: + """Decode 4 bytes as little-endian float.""" + if len(raw) == 4: + try: + val = struct.unpack(' Optional[float]: + """Decode 8 bytes as little-endian double.""" + if len(raw) == 8: + try: + val = struct.unpack(' Dict[str, Tuple[str, float]]: + """Scan a packet and return all numeric fields.""" + results = {} + if len(packet) < HEADER_SIZE + 10: + return results + + proto_data = packet[HEADER_SIZE:] + parser = ProtobufParser(proto_data) + fields = parser.parse_message(collect_repeated={14, 16, 20}) + + for field_num, val in fields.items(): + if isinstance(val, list): + for i, pf in enumerate(val): + scan_fields(pf, f"{field_num}[{i}]", results) + else: + scan_fields(val, f"{field_num}", results) + + return results + + +def find_parent_group(path: str) -> str: + """Extract parent field group from path (e.g., '3.1' -> '3').""" + parts = path.split('.') + return parts[0] if parts else path + + +def main(): + global running + + parser = argparse.ArgumentParser( + description="Find SOG/COG fields while at dock", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Expected patterns at dock: + SOG: ~0.0-0.2 kts (0-0.1 m/s) with small fluctuations + COG: Wildly jumping 0-359° (0-6.28 rad) due to GPS noise at low speed + +The script will identify fields matching these patterns and group them. + """ + ) + parser.add_argument('-i', '--interface', required=True, + help='Interface IP for Raymarine multicast (e.g., 198.18.5.5)') + parser.add_argument('-n', '--samples', type=int, default=30, + help='Number of samples to collect (default: 30)') + parser.add_argument('--interval', type=float, default=0.5, + help='Seconds between samples (default: 0.5)') + parser.add_argument('--sog-max', type=float, default=0.2, + help='Max expected SOG in knots at dock (default: 0.2)') + parser.add_argument('--cog-range', type=float, default=90, + help='Min expected COG range in degrees (default: 90)') + parser.add_argument('--verbose', '-v', action='store_true', + help='Show all fields, not just candidates') + + args = parser.parse_args() + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # Create sockets + sockets = [] + for group, port in MULTICAST_GROUPS: + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, 'SO_REUSEPORT'): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind(('', port)) + mreq = struct.pack("4s4s", socket.inet_aton(group), socket.inet_aton(args.interface)) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + sock.setblocking(False) + sockets.append((sock, group, port)) + except Exception as e: + print(f"Warning: Could not join {group}:{port}: {e}") + + if not sockets: + print("Error: Could not join any multicast groups") + sys.exit(1) + + print("=" * 70) + print("DOCK FINDER - SOG/COG Field Discovery") + print("=" * 70) + print(f"Joined {len(sockets)} multicast groups:") + for sock, group, port in sockets: + print(f" - {group}:{port}") + print() + print(f"Looking for fields at dock:") + print(f" SOG: 0.0 - {args.sog_max:.1f} kts (fluctuating near zero)") + print(f" COG: Jumping with range >= {args.cog_range}° (GPS noise at low speed)") + print(f"Collecting {args.samples} samples at {args.interval}s intervals...") + print("-" * 70) + + # Collect samples with diagnostics - track by packet size + field_data: Dict[str, FieldStats] = {} + field_data_by_size: Dict[int, Dict[str, FieldStats]] = defaultdict(dict) + samples_collected = 0 + last_sample_time_by_size: Dict[int, float] = defaultdict(float) + + # Diagnostic counters + packets_by_group: Dict[str, int] = defaultdict(int) + packets_by_size: Dict[int, int] = defaultdict(int) + empty_parse_count = 0 + total_packets = 0 + + try: + while running and samples_collected < args.samples: + for sock, group, port in sockets: + try: + data, addr = sock.recvfrom(65535) + pkt_size = len(data) + total_packets += 1 + packets_by_group[f"{group}:{port}"] += 1 + packets_by_size[pkt_size] += 1 + + now = time.time() + # Rate limit per packet size, not globally + if (now - last_sample_time_by_size[pkt_size]) < args.interval: + continue + + results = scan_packet(data) + if not results: + empty_parse_count += 1 + continue + + samples_collected += 1 + last_sample_time_by_size[pkt_size] = now + + # Update statistics (global and per-size) + for path, (wire_type, value) in results.items(): + # Global stats + if path not in field_data: + field_data[path] = FieldStats(path, wire_type, []) + field_data[path].values.append(value) + + # Per-size stats + size_fields = field_data_by_size[pkt_size] + if path not in size_fields: + size_fields[path] = FieldStats(path, wire_type, []) + size_fields[path].values.append(value) + + # Progress indicator + pct = min(100, (samples_collected / args.samples) * 100) + print(f"\r Collecting: {samples_collected}/{args.samples} ({pct:.0f}%) [pkts: {total_packets}]", end='', flush=True) + + except BlockingIOError: + continue + + time.sleep(0.01) + + finally: + for sock, _, _ in sockets: + sock.close() + + print() # Newline after progress + + # Show packet diagnostics + print() + print("*** PACKET DIAGNOSTICS ***") + print("-" * 70) + print(f" Total packets received: {total_packets}") + print(f" Packets with no parseable fields: {empty_parse_count}") + print() + print(" Packets by multicast group:") + for grp, cnt in sorted(packets_by_group.items(), key=lambda x: -x[1]): + print(f" {grp}: {cnt}") + print() + print(" Packets by size (top 10):") + for size, cnt in sorted(packets_by_size.items(), key=lambda x: -x[1])[:10]: + print(f" {size} bytes: {cnt}") + + # Show fields by packet size + print() + print("*** FIELDS BY PACKET SIZE ***") + print("-" * 70) + for pkt_size in sorted(field_data_by_size.keys(), reverse=True): + size_fields = field_data_by_size[pkt_size] + if not size_fields: + continue + field_paths = sorted(size_fields.keys()) + sample_count = max(s.count for s in size_fields.values()) if size_fields else 0 + print(f"\n {pkt_size} bytes ({sample_count} samples, {len(field_paths)} fields):") + for path in field_paths[:20]: # Show first 20 fields + stats = size_fields[path] + # Show with interpretation + interp = "" + if 0 <= stats.min_val and stats.max_val <= 7: + min_deg = (stats.min_val * RAD_TO_DEG) % 360 + max_deg = (stats.max_val * RAD_TO_DEG) % 360 + interp = f" | {min_deg:.1f}°-{max_deg:.1f}°" + elif 0 <= stats.min_val and stats.max_val <= 50: + min_kts = stats.min_val * MS_TO_KTS + max_kts = stats.max_val * MS_TO_KTS + interp = f" | {min_kts:.2f}-{max_kts:.2f} kts" + print(f" {path:<15} {stats.min_val:>10.4f} - {stats.max_val:>10.4f} (range: {stats.range_val:.4f}){interp}") + if len(field_paths) > 20: + print(f" ... and {len(field_paths) - 20} more fields") + + if samples_collected < 5: + print("\nError: Not enough samples collected. Check your network connection.") + sys.exit(1) + + # Analyze results - use per-packet-size data for better detection + print() + print("=" * 70) + print(f"ANALYSIS RESULTS ({samples_collected} samples)") + print("=" * 70) + + sog_candidates = [] + cog_candidates = [] + + # Analyze each packet size separately + for pkt_size, size_fields in field_data_by_size.items(): + for path, stats in size_fields.items(): + if stats.count < 3: # Need at least 3 samples + continue + + if stats.is_sog_candidate(): + # Add packet size info to path for clarity + stats.path = f"{path} ({pkt_size}B)" + sog_candidates.append(stats) + if stats.is_cog_candidate(): + stats.path = f"{path} ({pkt_size}B)" + cog_candidates.append(stats) + + # Print SOG candidates + print("\n*** POTENTIAL SOG FIELDS (speed near zero with fluctuation) ***") + print("-" * 70) + if sog_candidates: + for stats in sorted(sog_candidates, key=lambda s: s.std_dev, reverse=True): + min_kts, max_kts, mean_kts = stats.as_knots() + print(f" {stats.path}") + print(f" Raw (m/s): {stats.min_val:.4f} - {stats.max_val:.4f} " + f"(mean: {stats.mean:.4f}, std: {stats.std_dev:.4f})") + print(f" As knots: {min_kts:.3f} - {max_kts:.3f} " + f"(mean: {mean_kts:.3f})") + print() + else: + print(" (No candidates found)") + print(" Try increasing --sog-max or collecting more samples") + + # Print COG candidates + print("\n*** POTENTIAL COG FIELDS (angle jumping widely) ***") + print("-" * 70) + if cog_candidates: + for stats in sorted(cog_candidates, key=lambda s: s.range_val, reverse=True): + min_deg = (stats.min_val * RAD_TO_DEG) % 360 + max_deg = (stats.max_val * RAD_TO_DEG) % 360 + range_deg = stats.range_val * RAD_TO_DEG + print(f" {stats.path}") + print(f" Raw (rad): {stats.min_val:.4f} - {stats.max_val:.4f} " + f"(range: {stats.range_val:.2f} rad, std: {stats.std_dev:.2f})") + print(f" As degrees: {min_deg:.1f}° - {max_deg:.1f}° " + f"(range: {range_deg:.1f}°)") + print() + else: + print(" (No candidates found)") + print(" Try decreasing --cog-range or collecting more samples") + + # Look for paired candidates in the same parent group + print("\n*** PAIRED SOG/COG CANDIDATES (same field group) ***") + print("-" * 70) + + sog_groups = {find_parent_group(s.path): s for s in sog_candidates} + cog_groups = {find_parent_group(s.path): s for s in cog_candidates} + + common_groups = set(sog_groups.keys()) & set(cog_groups.keys()) + + if common_groups: + for group in sorted(common_groups): + sog = sog_groups[group] + cog = cog_groups[group] + min_kts, max_kts, _ = sog.as_knots() + range_deg = cog.range_val * RAD_TO_DEG + + print(f" Field Group {group}:") + print(f" SOG: {sog.path}") + print(f" {min_kts:.3f} - {max_kts:.3f} kts") + print(f" COG: {cog.path}") + print(f" Range: {range_deg:.1f}° (std: {cog.std_dev:.2f} rad)") + print() + else: + print(" No paired SOG/COG fields found in the same group") + print(" SOG and COG may be in different field groups") + + # Show sample values for top candidates + if sog_candidates or cog_candidates: + print("\n*** LAST 5 SAMPLE VALUES ***") + print("-" * 70) + + if sog_candidates: + top_sog = sorted(sog_candidates, key=lambda s: s.std_dev, reverse=True)[0] + print(f" Top SOG candidate ({top_sog.path}):") + last_vals = top_sog.values[-5:] + kts_vals = [v * MS_TO_KTS for v in last_vals] + print(f" m/s: {[f'{v:.4f}' for v in last_vals]}") + print(f" kts: {[f'{v:.3f}' for v in kts_vals]}") + + if cog_candidates: + top_cog = sorted(cog_candidates, key=lambda s: s.range_val, reverse=True)[0] + print(f" Top COG candidate ({top_cog.path}):") + last_vals = top_cog.values[-5:] + deg_vals = [(v * RAD_TO_DEG) % 360 for v in last_vals] + print(f" rad: {[f'{v:.4f}' for v in last_vals]}") + print(f" deg: {[f'{v:.1f}' for v in deg_vals]}") + + # Show diagnostic info if no candidates found, or if verbose + no_candidates = not sog_candidates and not cog_candidates + if args.verbose or no_candidates: + print("\n*** ALL NUMERIC FIELDS (diagnostic - per packet size) ***") + print("-" * 70) + print(f" {'Path':<25} {'Type':<5} {'Min':>12} {'Max':>12} {'Range':>12} {'StdDev':>10}") + print("-" * 70) + + # Collect all valid fields from per-packet-size data + all_fields = [] + for pkt_size, size_fields in sorted(field_data_by_size.items(), reverse=True): + for path, stats in size_fields.items(): + if stats.count < 3: + continue + # Create a copy with packet size in path + stats_copy = copy(stats) + stats_copy.path = f"{path} ({pkt_size}B)" + all_fields.append(stats_copy) + + # Sort by range (most variable first) + for stats in sorted(all_fields, key=lambda s: s.range_val, reverse=True): + print(f" {stats.path:<25} {stats.wire_type:<5} " + f"{stats.min_val:>12.4f} {stats.max_val:>12.4f} " + f"{stats.range_val:>12.4f} {stats.std_dev:>10.4f}") + + # Show interpretation hints for top variable fields + print("\n*** INTERPRETATION HINTS (top 10 most variable fields) ***") + print("-" * 70) + top_variable = sorted(all_fields, key=lambda s: s.range_val, reverse=True)[:10] + + for stats in top_variable: + print(f"\n {stats.path} ({stats.wire_type}):") + print(f" Raw: {stats.min_val:.6f} to {stats.max_val:.6f}") + + # Try angle interpretation (radians) + if 0 <= stats.min_val and stats.max_val <= 7: + min_deg = (stats.min_val * RAD_TO_DEG) % 360 + max_deg = (stats.max_val * RAD_TO_DEG) % 360 + range_deg = stats.range_val * RAD_TO_DEG + print(f" As angle (rad->deg): {min_deg:.1f}° to {max_deg:.1f}° (range: {range_deg:.1f}°)") + + # Try speed interpretation (m/s) + if 0 <= stats.min_val and stats.max_val <= 100: + min_kts = stats.min_val * MS_TO_KTS + max_kts = stats.max_val * MS_TO_KTS + print(f" As speed (m/s->kts): {min_kts:.3f} to {max_kts:.3f} kts") + + # Try temperature interpretation (Kelvin) + if 250 <= stats.min_val <= 350: + min_c = stats.min_val - 273.15 + max_c = stats.max_val - 273.15 + print(f" As temp (K->°C): {min_c:.1f}°C to {max_c:.1f}°C") + + # GPS coordinate check + if -180 <= stats.min_val <= 180 and stats.range_val < 1: + print(f" Could be GPS coordinate (low variance)") + + if no_candidates: + print("\n" + "=" * 70) + print("SUGGESTIONS:") + print("=" * 70) + print(" No automatic matches found. Look at the fields above for:") + print(" - SOG: Small values (< 0.5 m/s) with some variance") + print(" - COG: Values in 0-6.28 range (radians) with HIGH variance") + print() + print(" Common issues:") + print(" - GPS may not have lock (check for lat/lon)") + print(" - Values may be in different units than expected") + print(" - Try: --sog-max 1.0 --cog-range 45") + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/debug/field5_study.py b/axiom-nmea/debug/field5_study.py new file mode 100755 index 0000000..5b92cb8 --- /dev/null +++ b/axiom-nmea/debug/field5_study.py @@ -0,0 +1,412 @@ +#!/usr/bin/env python3 +""" +Field 5 Study - Comprehensive analysis of Field 5 subfields. + +Field 5 appears to contain SOG/COG data based on dock testing. +This script collects extensive samples to document all subfields. + +Usage: + python field5_study.py -i 198.18.5.5 + python field5_study.py -i 198.18.5.5 --samples 100 --interval 0.2 +""" + +import argparse +import math +import os +import signal +import socket +import struct +import sys +import time +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Dict, List, Optional, Tuple + +# Add parent directory to path for library import +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from raymarine_nmea.protocol.parser import ProtobufParser, ProtoField +from raymarine_nmea.protocol.constants import ( + WIRE_VARINT, WIRE_FIXED64, WIRE_LENGTH, WIRE_FIXED32, + HEADER_SIZE, RAD_TO_DEG, MS_TO_KTS, +) +from raymarine_nmea.sensors import MULTICAST_GROUPS + +running = True + + +def signal_handler(signum, frame): + global running + running = False + + +@dataclass +class FieldStats: + """Statistics for a field across multiple samples.""" + path: str + wire_type: str + values: List[float] = field(default_factory=list) + + @property + def count(self) -> int: + return len(self.values) + + @property + def min_val(self) -> float: + return min(self.values) if self.values else 0 + + @property + def max_val(self) -> float: + return max(self.values) if self.values else 0 + + @property + def range_val(self) -> float: + return self.max_val - self.min_val + + @property + def mean(self) -> float: + return sum(self.values) / len(self.values) if self.values else 0 + + @property + def std_dev(self) -> float: + if len(self.values) < 2: + return 0 + mean = self.mean + variance = sum((v - mean) ** 2 for v in self.values) / len(self.values) + return math.sqrt(variance) + + +def decode_float(raw: bytes) -> Optional[float]: + """Decode 4 bytes as little-endian float.""" + if len(raw) == 4: + try: + val = struct.unpack(' Optional[float]: + """Decode 8 bytes as little-endian double.""" + if len(raw) == 8: + try: + val = struct.unpack(' Dict[str, Tuple[str, float]]: + """Extract all Field 5 subfields from a packet.""" + results = {} + if len(packet) < HEADER_SIZE + 10: + return results + + proto_data = packet[HEADER_SIZE:] + parser = ProtobufParser(proto_data) + fields = parser.parse_message() + + # Look for Field 5 + if 5 not in fields: + return results + + field5 = fields[5] + + # If Field 5 is a nested message, extract children + if field5.children: + for child_num, child in field5.children.items(): + path = f"5.{child_num}" + + if child.wire_type == WIRE_FIXED32: + val = decode_float(child.value) + if val is not None: + results[path] = ('f32', val) + + elif child.wire_type == WIRE_FIXED64: + val = decode_double(child.value) + if val is not None: + results[path] = ('f64', val) + + elif child.wire_type == WIRE_VARINT: + results[path] = ('var', float(child.value)) + + # Check for deeper nesting + if child.children: + for subchild_num, subchild in child.children.items(): + subpath = f"5.{child_num}.{subchild_num}" + if subchild.wire_type == WIRE_FIXED32: + val = decode_float(subchild.value) + if val is not None: + results[subpath] = ('f32', val) + elif subchild.wire_type == WIRE_FIXED64: + val = decode_double(subchild.value) + if val is not None: + results[subpath] = ('f64', val) + + # Field 5 itself might be a scalar + elif field5.wire_type == WIRE_FIXED32: + val = decode_float(field5.value) + if val is not None: + results['5'] = ('f32', val) + + elif field5.wire_type == WIRE_FIXED64: + val = decode_double(field5.value) + if val is not None: + results['5'] = ('f64', val) + + return results + + +def interpret_value(val: float, wire_type: str) -> Dict[str, str]: + """Generate possible interpretations of a value.""" + interps = {} + + # Angle (radians to degrees) + if 0 <= val <= 2 * math.pi + 0.5: + deg = (val * RAD_TO_DEG) % 360 + interps['angle'] = f"{deg:.1f}°" + + # Speed (m/s to knots) + if 0 <= val <= 100: + kts = val * MS_TO_KTS + interps['speed'] = f"{kts:.2f} kts" + + # Small angle (degrees already) + if 0 <= val <= 360: + interps['deg_direct'] = f"{val:.1f}° (if already degrees)" + + # Temperature (Kelvin) + if 250 <= val <= 350: + c = val - 273.15 + interps['temp'] = f"{c:.1f}°C" + + return interps + + +def main(): + global running + + parser = argparse.ArgumentParser( + description="Study Field 5 subfields comprehensively", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument('-i', '--interface', required=True, + help='Interface IP for Raymarine multicast') + parser.add_argument('-n', '--samples', type=int, default=50, + help='Number of samples to collect (default: 50)') + parser.add_argument('--interval', type=float, default=0.3, + help='Seconds between samples (default: 0.3)') + + args = parser.parse_args() + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # Create sockets + sockets = [] + for group, port in MULTICAST_GROUPS: + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, 'SO_REUSEPORT'): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind(('', port)) + mreq = struct.pack("4s4s", socket.inet_aton(group), socket.inet_aton(args.interface)) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + sock.setblocking(False) + sockets.append((sock, group, port)) + except Exception as e: + print(f"Warning: Could not join {group}:{port}: {e}") + + if not sockets: + print("Error: Could not join any multicast groups") + sys.exit(1) + + print("=" * 80) + print("FIELD 5 COMPREHENSIVE STUDY") + print("=" * 80) + print(f"Collecting {args.samples} samples at {args.interval}s intervals...") + print("-" * 80) + + # Track by packet size + field5_by_size: Dict[int, Dict[str, FieldStats]] = defaultdict(dict) + packets_with_field5 = 0 + packets_without_field5 = 0 + total_packets = 0 + last_sample_time_by_size: Dict[int, float] = defaultdict(float) + + try: + samples_collected = 0 + while running and samples_collected < args.samples: + for sock, group, port in sockets: + try: + data, addr = sock.recvfrom(65535) + pkt_size = len(data) + total_packets += 1 + + now = time.time() + if (now - last_sample_time_by_size[pkt_size]) < args.interval: + continue + + results = extract_field5(data) + + if results: + packets_with_field5 += 1 + samples_collected += 1 + last_sample_time_by_size[pkt_size] = now + + for path, (wire_type, value) in results.items(): + size_fields = field5_by_size[pkt_size] + if path not in size_fields: + size_fields[path] = FieldStats(path, wire_type, []) + size_fields[path].values.append(value) + + pct = (samples_collected / args.samples) * 100 + print(f"\r Collecting: {samples_collected}/{args.samples} ({pct:.0f}%)", end='', flush=True) + else: + packets_without_field5 += 1 + + except BlockingIOError: + continue + + time.sleep(0.01) + + finally: + for sock, _, _ in sockets: + sock.close() + + print() + print() + + # Summary + print("=" * 80) + print("FIELD 5 STUDY RESULTS") + print("=" * 80) + print(f" Total packets scanned: {total_packets}") + print(f" Packets with Field 5: {packets_with_field5}") + print(f" Packets without Field 5: {packets_without_field5}") + print() + + if not field5_by_size: + print(" No Field 5 data found!") + sys.exit(1) + + # Show Field 5 structure by packet size + print("=" * 80) + print("FIELD 5 SUBFIELDS BY PACKET SIZE") + print("=" * 80) + + all_subfields = set() + for pkt_size in sorted(field5_by_size.keys()): + size_fields = field5_by_size[pkt_size] + all_subfields.update(size_fields.keys()) + + # For each packet size that has Field 5 + for pkt_size in sorted(field5_by_size.keys()): + size_fields = field5_by_size[pkt_size] + if not size_fields: + continue + + sample_count = max(s.count for s in size_fields.values()) + print(f"\n--- {pkt_size} bytes ({sample_count} samples) ---") + print() + + for path in sorted(size_fields.keys(), key=lambda x: [int(p) for p in x.split('.')]): + stats = size_fields[path] + + print(f" {path} ({stats.wire_type}):") + print(f" Samples: {stats.count}") + print(f" Range: {stats.min_val:.6f} to {stats.max_val:.6f}") + print(f" Mean: {stats.mean:.6f}") + print(f" StdDev: {stats.std_dev:.6f}") + + # Show interpretations + interps = interpret_value(stats.mean, stats.wire_type) + if interps: + print(f" Interpretations:") + for itype, ival in interps.items(): + print(f" - As {itype}: {ival}") + + # Behavioral analysis + if stats.std_dev < 0.001 and stats.count >= 3: + print(f" Behavior: CONSTANT") + elif stats.range_val > 3.0 and 0 <= stats.min_val <= 7: + range_deg = stats.range_val * RAD_TO_DEG + print(f" Behavior: HIGHLY VARIABLE ({range_deg:.0f}° range) - likely COG or heading") + elif stats.range_val > 0.01 and stats.max_val < 1.0: + range_kts = stats.range_val * MS_TO_KTS + print(f" Behavior: SMALL FLUCTUATION ({range_kts:.3f} kts range) - could be SOG") + + print() + + # Summary table + print("=" * 80) + print("FIELD 5 SUMMARY TABLE") + print("=" * 80) + print() + print(f" {'Subfield':<10} {'Type':<5} {'Min':>12} {'Max':>12} {'StdDev':>10} {'Behavior':<20} {'Likely Purpose'}") + print("-" * 95) + + # Aggregate across all packet sizes for the summary + aggregated: Dict[str, FieldStats] = {} + for pkt_size, size_fields in field5_by_size.items(): + for path, stats in size_fields.items(): + if path not in aggregated: + aggregated[path] = FieldStats(path, stats.wire_type, []) + aggregated[path].values.extend(stats.values) + + for path in sorted(aggregated.keys(), key=lambda x: [int(p) for p in x.split('.')]): + stats = aggregated[path] + + # Determine behavior + if stats.std_dev < 0.001: + behavior = "Constant" + elif stats.range_val > 3.0: + behavior = f"Variable ({stats.range_val * RAD_TO_DEG:.0f}° range)" + elif stats.range_val > 0.01: + behavior = f"Fluctuating" + else: + behavior = "Near-constant" + + # Guess purpose based on behavior and value range + purpose = "Unknown" + if stats.range_val > 3.0 and 0 <= stats.min_val <= 7: + purpose = "COG or heading" + elif stats.std_dev < 0.001 and 0.005 <= stats.mean <= 0.5: + purpose = "SOG (at dock)" + elif stats.std_dev < 0.001 and stats.mean > 10: + purpose = "Fixed parameter" + elif 0 <= stats.mean <= 0.2 and stats.max_val < 1: + purpose = "SOG candidate" + + print(f" {path:<10} {stats.wire_type:<5} {stats.min_val:>12.4f} {stats.max_val:>12.4f} " + f"{stats.std_dev:>10.4f} {behavior:<20} {purpose}") + + print() + print("=" * 80) + print("INTERPRETATION GUIDE") + print("=" * 80) + print(""" + Based on dock behavior (SOG ~0, COG jumping wildly): + + - COG (Course Over Ground): Look for fields with HIGH variance spanning + most of 0-2π radians (~0-360°). At dock, GPS-derived COG is unreliable + and jumps randomly. + + - SOG (Speed Over Ground): Look for fields with small values (~0.01-0.1 m/s) + that are relatively constant at dock. May show slight fluctuation. + + - Heading: May be similar to COG but derived from compass, so more stable. + + - Fixed parameters: Constants like 0.05, 0.1, 11.93 may be configuration + values, damping factors, or display settings. +""") + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/debug/field_debugger.py b/axiom-nmea/debug/field_debugger.py new file mode 100644 index 0000000..0882589 --- /dev/null +++ b/axiom-nmea/debug/field_debugger.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +""" +Field Debugger - Shows all protobuf fields in columns for mapping real-world values. + +Displays each top-level field as a column, with subfields as rows. +Updates every few seconds to show value progression over time. + +Usage: + python3 field_debugger.py -i 192.168.1.100 # Live capture + python3 field_debugger.py --pcap capture.pcap # From file + python3 field_debugger.py --pcap capture.pcap -n 5 # Show 5 snapshots +""" + +import struct +import socket +import time +import argparse +import threading +import sys +from datetime import datetime +from collections import defaultdict +from typing import Dict, List, Any, Optional + +# Wire types +WIRE_VARINT = 0 +WIRE_FIXED64 = 1 +WIRE_LENGTH = 2 +WIRE_FIXED32 = 5 + +HEADER_SIZE = 20 + +MULTICAST_GROUPS = [ + ("226.192.206.98", 2561), + ("226.192.206.99", 2562), + ("226.192.206.100", 2563), + ("226.192.206.101", 2564), + ("226.192.206.102", 2565), + ("226.192.219.0", 3221), + ("239.2.1.1", 2154), # May contain tank/engine data +] + + +class ProtobufParser: + """Parse protobuf without schema.""" + + def __init__(self, data: bytes): + self.data = data + self.pos = 0 + + def read_varint(self) -> int: + result = 0 + shift = 0 + while self.pos < len(self.data): + byte = self.data[self.pos] + self.pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + return result + + def parse(self) -> Dict[int, Any]: + """Parse message, return dict of field_num -> (wire_type, value, children).""" + fields = {} + while self.pos < len(self.data): + try: + start = self.pos + tag = self.read_varint() + field_num = tag >> 3 + wire_type = tag & 0x07 + + if field_num == 0 or field_num > 1000: + break + + if wire_type == WIRE_VARINT: + value = self.read_varint() + children = None + elif wire_type == WIRE_FIXED64: + value = self.data[self.pos:self.pos + 8] + self.pos += 8 + children = None + elif wire_type == WIRE_LENGTH: + length = self.read_varint() + value = self.data[self.pos:self.pos + length] + self.pos += length + # Try to parse as nested + try: + nested = ProtobufParser(value) + children = nested.parse() + if nested.pos < len(value) * 0.5: + children = None + except: + children = None + elif wire_type == WIRE_FIXED32: + value = self.data[self.pos:self.pos + 4] + self.pos += 4 + children = None + else: + break + + fields[field_num] = (wire_type, value, children) + except: + break + return fields + + +# Known field labels from reverse engineering +FIELD_LABELS = { + # Top-level fields + (1,): "DeviceInfo", + (2,): "GPS", + (3,): "HeadingBlock", + (7,): "DepthBlock", + (8,): "RateOfTurn", + (10,): "Unknown10", + (12,): "Unknown12", + (13,): "WindNav", + (14,): "SensorData", + (21,): "Angles", + + # Field 1 subfields (Device Info) + (1, 1): "DeviceName", + (1, 2): "SerialInfo", + + # Field 2 subfields (GPS) + (2, 1): "LATITUDE", + (2, 2): "LONGITUDE", + (2, 3): "Unknown", + (2, 4): "Altitude?", + (2, 5): "Timestamp?", + (2, 6): "Distance?", + + # Field 3 subfields (Heading) + (3, 1): "HeadingRaw", + (3, 2): "HEADING", + + # Field 7 subfields (Depth) - only in larger packets (1472B+) + (7, 1): "DEPTH_M", # Depth in METERS + + # Field 8 subfields + (8, 1): "ROT?", + (8, 2): "Unknown", + + # Field 13 subfields (Wind/Navigation) - MAIN SENSOR BLOCK + (13, 1): "Heading1", + (13, 2): "Heading2", + (13, 3): "SmallAngle", + (13, 4): "TWD", # True Wind Direction + (13, 5): "TWS", # True Wind Speed + (13, 6): "AWS", # Apparent Wind Speed? + (13, 7): "AWD?", # Apparent Wind Direction? + (13, 8): "Heading1_dup", + (13, 9): "Heading2_dup", + (13, 10): "SmallAngle_dup", + (13, 11): "TWS_dup", + (13, 12): "AWS_dup", + (13, 13): "AWD_dup?", + + # Field 21 subfields + (21, 1): "Unknown", + (21, 2): "Angle1", + (21, 3): "Unknown", + (21, 4): "Unknown", +} + + +def get_label(field_path: tuple) -> str: + """Get label for a field path, or empty string if unknown.""" + return FIELD_LABELS.get(field_path, "") + + +def format_value(wire_type: int, value: Any) -> str: + """Format a protobuf value for display.""" + if wire_type == WIRE_VARINT: + if value > 2**31: + return f"v:{value} (0x{value:x})" + return f"v:{value}" + + elif wire_type == WIRE_FIXED64: + try: + d = struct.unpack(' 10000: + return f"d:{d:.1f}" + if -180 <= d <= 180: + return f"d:{d:.6f}" + return f"d:{d:.2f}" + except: + return f"x:{value.hex()[:16]}" + + elif wire_type == WIRE_FIXED32: + try: + f = struct.unpack(' 15: + return f's:"{s[:12]}..."' + return f's:"{s}"' + except: + pass + return f"[{len(value)}B]" + + return "?" + + +def extract_fields(packet: bytes) -> Optional[Dict[int, Any]]: + """Extract all fields from a packet.""" + if len(packet) < HEADER_SIZE + 10: + return None + + proto_data = packet[HEADER_SIZE:] + parser = ProtobufParser(proto_data) + return parser.parse() + + +def print_snapshot(fields: Dict[int, Any], timestamp: str, packet_size: int): + """Print a snapshot of all fields in columnar format.""" + # Build column data + columns = {} # field_num -> list of (subfield_path, value_str) + + def process_field(field_num: int, wire_type: int, value: Any, children: Optional[Dict], prefix: str = ""): + col_key = field_num + if col_key not in columns: + columns[col_key] = [] + + if children: + # Has subfields + columns[col_key].append((f"{prefix}", "[msg]")) + for sub_num, (sub_wt, sub_val, sub_children) in sorted(children.items()): + label = get_label((field_num, sub_num)) + label_str = f" {label}" if label else "" + val_str = format_value(sub_wt, sub_val) + columns[col_key].append((f" .{sub_num}{label_str}", val_str)) + + # Go one level deeper for nested messages + if sub_children: + for subsub_num, (subsub_wt, subsub_val, _) in sorted(sub_children.items()): + val_str2 = format_value(subsub_wt, subsub_val) + columns[col_key].append((f" .{sub_num}.{subsub_num}", val_str2)) + else: + val_str = format_value(wire_type, value) + columns[col_key].append((prefix or "val", val_str)) + + # Process all top-level fields + for field_num, (wire_type, value, children) in sorted(fields.items()): + process_field(field_num, wire_type, value, children) + + # Print header + print("\n" + "=" * 100) + print(f" {timestamp} | Packet: {packet_size} bytes | Fields: {len(fields)}") + print("=" * 100) + + # Determine column layout + col_nums = sorted(columns.keys()) + if not col_nums: + print(" No fields decoded") + return + + # Calculate column widths + col_width = 28 + cols_per_row = min(4, len(col_nums)) + + # Print columns in groups + for start_idx in range(0, len(col_nums), cols_per_row): + group_cols = col_nums[start_idx:start_idx + cols_per_row] + + # Header row with labels + header = "" + for col_num in group_cols: + label = get_label((col_num,)) + if label: + hdr_text = f"F{col_num} {label}" + else: + hdr_text = f"Field {col_num}" + header += f"| {hdr_text:<{col_width - 1}}" + print(header + "|") + print("-" * (len(group_cols) * (col_width + 2) + 1)) + + # Find max rows needed + max_rows = max(len(columns[c]) for c in group_cols) + + # Print rows + for row_idx in range(max_rows): + row = "" + for col_num in group_cols: + col_data = columns[col_num] + if row_idx < len(col_data): + path, val = col_data[row_idx] + cell = f"{path}: {val}" + if len(cell) > col_width - 1: + cell = cell[:col_width - 4] + "..." + row += f"| {cell:<{col_width - 1}}" + else: + row += f"| {'':<{col_width - 1}}" + print(row + "|") + + print() + + +def read_pcap(filename: str) -> List[bytes]: + """Read packets from pcap file.""" + packets = [] + with open(filename, 'rb') as f: + header = f.read(24) + magic = struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append((ts_sec + ts_usec / 1e6, pkt_data[payload_start:])) + return packets + + +class LiveListener: + """Listen for live packets.""" + + def __init__(self, interface_ip: str): + self.interface_ip = interface_ip + self.running = False + self.packets_by_group = {} # (group, port) -> (packet, size) + self.lock = threading.Lock() + + def _create_socket(self, group: str, port: int): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, 'SO_REUSEPORT'): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind(('', port)) + mreq = struct.pack("4s4s", socket.inet_aton(group), socket.inet_aton(self.interface_ip)) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + sock.settimeout(1.0) + return sock + + def _listen(self, sock, group: str, port: int): + key = (group, port) + while self.running: + try: + data, _ = sock.recvfrom(65535) + # Keep packets with protobuf payload (header + minimal data) + if len(data) >= 40: + with self.lock: + self.packets_by_group[key] = data + except socket.timeout: + continue + except: + pass + + def start(self): + self.running = True + for group, port in MULTICAST_GROUPS: + try: + sock = self._create_socket(group, port) + t = threading.Thread(target=self._listen, args=(sock, group, port), daemon=True) + t.start() + print(f"Listening on {group}:{port}") + except Exception as e: + print(f"Error: {e}") + + def get_all_packets(self) -> Dict[tuple, bytes]: + """Return dict of (group, port) -> packet for all groups with data.""" + with self.lock: + return dict(self.packets_by_group) + + def stop(self): + self.running = False + + +def main(): + parser = argparse.ArgumentParser(description="Field Debugger - Map protobuf fields to real values") + parser.add_argument('-i', '--interface', help='Interface IP for live capture') + parser.add_argument('--pcap', help='Read from pcap file') + parser.add_argument('-n', '--num-snapshots', type=int, default=10, help='Number of snapshots to show') + parser.add_argument('-t', '--interval', type=float, default=3.0, help='Seconds between snapshots') + parser.add_argument('-s', '--size', type=int, help='Only show packets of this size') + args = parser.parse_args() + + if not args.pcap and not args.interface: + parser.error("Either --interface or --pcap required") + + print("Field Debugger - Protobuf Field Mapper") + print("=" * 50) + print("Legend:") + print(" v:N = varint (integer)") + print(" d:N = double (64-bit float)") + print(" f:N (X°) = float as radians -> degrees") + print(" f:N (Xkt) = float as m/s -> knots") + print(" s:\"...\" = string") + print(" [NB] = N bytes (nested message)") + print("=" * 50) + + if args.pcap: + # Read from pcap + print(f"\nReading {args.pcap}...") + packets = read_pcap(args.pcap) + print(f"Loaded {len(packets)} packets") + + # Filter by size if requested + if args.size: + packets = [(ts, p) for ts, p in packets if len(p) == args.size] + print(f"Filtered to {len(packets)} packets of size {args.size}") + + # Group by size + by_size = defaultdict(list) + for ts, pkt in packets: + by_size[len(pkt)].append((ts, pkt)) + + print(f"\nPacket sizes: {sorted(by_size.keys())}") + + # Show snapshots from packets with sensor data + target_sizes = [s for s in sorted(by_size.keys()) if s >= 300] + if not target_sizes: + print("No packets >= 300 bytes found") + return + + # Pick largest sensor packets + target_size = target_sizes[-1] if not args.size else args.size + target_packets = by_size.get(target_size, []) + + if not target_packets: + print(f"No packets of size {target_size}") + return + + # Show snapshots at intervals through the capture + step = max(1, len(target_packets) // args.num_snapshots) + for i in range(0, len(target_packets), step): + if i // step >= args.num_snapshots: + break + ts, pkt = target_packets[i] + fields = extract_fields(pkt) + if fields: + timestamp = datetime.fromtimestamp(ts).strftime("%H:%M:%S.%f")[:-3] + print_snapshot(fields, timestamp, len(pkt)) + + else: + # Live capture + listener = LiveListener(args.interface) + listener.start() + + print(f"\nShowing {args.num_snapshots} snapshots, {args.interval}s apart") + print("Press Ctrl+C to stop\n") + + try: + for i in range(args.num_snapshots): + time.sleep(args.interval) + all_packets = listener.get_all_packets() + if all_packets: + timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3] + for (group, port), pkt in sorted(all_packets.items()): + if args.size and len(pkt) != args.size: + continue + fields = extract_fields(pkt) + if fields: + header = f"{group}:{port}" + print_snapshot(fields, f"{timestamp} [{header}]", len(pkt)) + else: + print(f"[{i+1}] No packets received yet...") + except KeyboardInterrupt: + print("\nStopped") + finally: + listener.stop() + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/debug/field_mapping.py b/axiom-nmea/debug/field_mapping.py new file mode 100644 index 0000000..073e47b --- /dev/null +++ b/axiom-nmea/debug/field_mapping.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +""" +Extract and display the field mapping for sensor data. +Based on protobuf structure analysis. +""" + +import struct + +def read_pcap(filename): + packets = [] + with open(filename, 'rb') as f: + header = f.read(24) + magic = struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + +# Based on protobuf analysis, here's the structure: +STRUCTURE = """ +================================================================================ +RAYMARINE PACKET STRUCTURE (from protobuf analysis) +================================================================================ + +FIXED HEADER (20 bytes @ 0x0000-0x0013): + 0x0000-0x0007: Packet identifier (00 00 00 00 00 00 00 01) + 0x0008-0x000B: Source ID + 0x000C-0x000F: Message type indicator + 0x0010-0x0013: Payload length + +PROTOBUF MESSAGE (starts @ 0x0014): + + Field 1 (length-delim): Device Info + └─ Field 1: Device name ("AXIOM 12") + └─ Field 2: Serial number info + + Field 2 (length-delim): GPS/Position Data + ├─ Field 1 (fixed64/double): LATITUDE + ├─ Field 2 (fixed64/double): LONGITUDE + ├─ Field 3 (fixed64/double): (unknown, often NaN) + ├─ Field 4 (fixed64/double): (altitude or similar) + ├─ Field 5 (fixed64/double): (timestamp or distance) + └─ Field 6 (fixed64/double): (timestamp or distance) + + Field 3 (length-delim): Heading Block + ├─ Field 1 (fixed32/float): Heading value 1 (radians) + └─ Field 2 (fixed32/float): HEADING (radians) ← ~31-33° + + Field 6 (length-delim): [only in 446+ byte packets] + └─ Field 1 (fixed32/float): (unknown angle) + + Field 8 (length-delim): Rate/Motion Data + ├─ Field 1 (fixed32/float): Rate of turn? + └─ Field 2 (fixed32/float): (often NaN) + + Field 13 (length-delim): WIND/NAVIGATION DATA ← Main sensor block + ├─ Field 1 (fixed32/float): Heading copy (radians) + ├─ Field 2 (fixed32/float): Heading smoothed (radians) + ├─ Field 3 (fixed32/float): (small angle, ~0°) + ├─ Field 4 (fixed32/float): TRUE WIND DIRECTION (radians) ← ~62-70° + ├─ Field 5 (fixed32/float): WIND SPEED (m/s) ← ~7.26 = 14.1 kts + ├─ Field 6 (fixed32/float): Wind speed (different value) + ├─ Field 7 (fixed32/float): (large angle ~245°) + ├─ Field 8-13: Duplicates/smoothed values of above + + Field 14 (length-delim): Additional sensor data (multiple instances) + Field 21 (length-delim): More angles + Field 38, 41: Empty/reserved + +================================================================================ +KEY SENSOR MAPPINGS +================================================================================ + +SENSOR PARENT FIELD CHILD FIELD UNIT +------------------ ------------- ----------- -------- +Latitude Field 2 Field 1 degrees (double) +Longitude Field 2 Field 2 degrees (double) +Heading Field 3 Field 2 radians (float) +True Wind Dir Field 13 Field 4 radians (float) +Wind Speed Field 13 Field 5 m/s (float) + +================================================================================ +""" + +print(STRUCTURE) + +# Now verify with actual data +print("VERIFICATION WITH ACTUAL DATA:") +print("=" * 70) + +packets = read_pcap("raymarine_sample_TWD_62-70_HDG_29-35.pcap") + +# Get a 344-byte packet +for pkt in packets: + if len(pkt) == 344: + # Skip 20-byte header, protobuf starts at 0x14 + proto = pkt[0x14:] + + print(f"\n344-byte packet analysis:") + print(f" Expected: TWD 62-70°, Heading 29-35°") + + # Field 3 starts around offset 0x54 in original packet = 0x40 in proto + # But we need to navigate by protobuf structure + + # Let's use the known byte offsets and verify + # From earlier analysis: + # - 0x0070 had heading ~32.6° + # - 0x00a0 had TWD ~61.7° + + def get_float(data, offset): + if offset + 4 <= len(data): + return struct.unpack(' Optional[float]: + """Decode 4 bytes as float.""" + if len(raw) == 4: + try: + val = struct.unpack(' Optional[float]: + """Decode 8 bytes as double.""" + if len(raw) == 8: + try: + val = struct.unpack(' Dict[str, float]: + """Get all possible interpretations of a numeric value.""" + interps = {} + + # Angle interpretations (radians to degrees) + if 0 <= val <= 6.5: + interps['deg'] = (val * RAD_TO_DEG) % 360 + + # Speed interpretations (m/s to knots) + if 0 <= val <= 100: + interps['kts'] = val * MS_TO_KTS + + return interps + + +def check_cog_match(val: float, cog_min: float, cog_max: float) -> Optional[float]: + """Check if value could be COG in radians. Returns degrees or None.""" + if 0 <= val <= 6.5: # Valid radian range + deg = (val * RAD_TO_DEG) % 360 + # Handle wrap-around + if cog_min <= cog_max: + if cog_min <= deg <= cog_max: + return deg + else: # wrap-around case like 350-10 + if deg >= cog_min or deg <= cog_max: + return deg + return None + + +def check_sog_match(val: float, sog_min: float, sog_max: float) -> Optional[float]: + """Check if value could be SOG in m/s. Returns knots or None.""" + if 0 <= val <= 50: # Reasonable m/s range + kts = val * MS_TO_KTS + if sog_min <= kts <= sog_max: + return kts + return None + + +def scan_all_fields(pf: ProtoField, path: str, results: List[Dict]): + """Recursively scan ALL fields and collect numeric values.""" + + if pf.wire_type == WIRE_FIXED32: + val = decode_float(pf.value) + if val is not None: + interps = get_interpretations(val) + results.append({ + 'path': path, + 'wire': 'f32', + 'raw': val, + 'interps': interps + }) + + elif pf.wire_type == WIRE_FIXED64: + val = decode_double(pf.value) + if val is not None: + interps = get_interpretations(val) + results.append({ + 'path': path, + 'wire': 'f64', + 'raw': val, + 'interps': interps + }) + + elif pf.wire_type == WIRE_VARINT: + val = float(pf.value) + interps = get_interpretations(val) + results.append({ + 'path': path, + 'wire': 'var', + 'raw': pf.value, + 'interps': interps + }) + + # Recurse into children + if pf.children: + for child_num, child in pf.children.items(): + scan_all_fields(child, f"{path}.{child_num}", results) + + +def scan_packet(packet: bytes) -> List[Dict]: + """Scan a packet and return ALL numeric fields.""" + results = [] + if len(packet) < HEADER_SIZE + 10: + return results + + proto_data = packet[HEADER_SIZE:] + parser = ProtobufParser(proto_data) + fields = parser.parse_message(collect_repeated={14, 16, 20}) + + for field_num, val in fields.items(): + if isinstance(val, list): + for i, pf in enumerate(val): + scan_all_fields(pf, f"{field_num}[{i}]", results) + else: + scan_all_fields(val, f"{field_num}", results) + + return results + + +def main(): + global running + + parser = argparse.ArgumentParser(description="Find COG/SOG fields in Raymarine packets") + parser.add_argument('-i', '--interface', required=True, + help='Interface IP for Raymarine multicast (e.g., 198.18.5.5)') + parser.add_argument('--cog-min', type=float, default=0, + help='Minimum expected COG in degrees (default: 0)') + parser.add_argument('--cog-max', type=float, default=359, + help='Maximum expected COG in degrees (default: 359)') + parser.add_argument('--sog-min', type=float, default=0, + help='Minimum expected SOG in knots (default: 0)') + parser.add_argument('--sog-max', type=float, default=2.0, + help='Maximum expected SOG in knots (default: 2.0)') + parser.add_argument('-n', '--count', type=int, default=5, + help='Number of packets to analyze (default: 5)') + parser.add_argument('--interval', type=float, default=1.0, + help='Minimum interval between packets (default: 1.0)') + + args = parser.parse_args() + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # Create sockets + sockets = [] + for group, port in MULTICAST_GROUPS: + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, 'SO_REUSEPORT'): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind(('', port)) + mreq = struct.pack("4s4s", socket.inet_aton(group), socket.inet_aton(args.interface)) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + sock.setblocking(False) + sockets.append((sock, group, port)) + except Exception as e: + print(f"Error joining {group}:{port}: {e}") + + if not sockets: + print("Error: Could not join any multicast groups") + sys.exit(1) + + print(f"COG/SOG Field Finder") + print(f"====================") + print(f"Looking for COG: {args.cog_min}° - {args.cog_max}°") + print(f"Looking for SOG: {args.sog_min} - {args.sog_max} kts") + print(f"Analyzing {args.count} packets...") + print() + + # Track all field values across packets + field_values: Dict[str, List[Tuple[float, Dict]]] = {} + + packet_count = 0 + analyzed = 0 + last_time = 0 + + try: + while running and analyzed < args.count: + for sock, group, port in sockets: + try: + data, addr = sock.recvfrom(65535) + packet_count += 1 + + now = time.time() + if args.interval > 0 and (now - last_time) < args.interval: + continue + + results = scan_packet(data) + if not results: + continue + + analyzed += 1 + last_time = now + + # Collect values + for r in results: + path = r['path'] + if path not in field_values: + field_values[path] = [] + field_values[path].append((r['raw'], r['interps'])) + + except BlockingIOError: + continue + + time.sleep(0.01) + + finally: + for sock, _, _ in sockets: + sock.close() + + # Analyze and display results + print(f"\n{'='*80}") + print(f"ANALYSIS - {analyzed} packets") + print(f"{'='*80}") + + cog_candidates = [] + sog_candidates = [] + other_fields = [] + + for path in sorted(field_values.keys()): + values = field_values[path] + raw_vals = [v[0] for v in values] + + if not raw_vals: + continue + + min_raw = min(raw_vals) + max_raw = max(raw_vals) + avg_raw = sum(raw_vals) / len(raw_vals) + + # Check if this could be COG (radians -> degrees in range) + cog_matches = 0 + cog_degs = [] + for raw, interps in values: + if 'deg' in interps: + deg = interps['deg'] + cog_degs.append(deg) + cog_match = check_cog_match(raw, args.cog_min, args.cog_max) + if cog_match is not None: + cog_matches += 1 + + # Check if this could be SOG (m/s -> knots in range) + sog_matches = 0 + sog_kts = [] + for raw, interps in values: + if 'kts' in interps: + kts = interps['kts'] + sog_kts.append(kts) + sog_match = check_sog_match(raw, args.sog_min, args.sog_max) + if sog_match is not None: + sog_matches += 1 + + # Categorize + if cog_matches == len(values) and cog_degs: + cog_candidates.append((path, cog_degs, raw_vals)) + elif sog_matches == len(values) and sog_kts: + sog_candidates.append((path, sog_kts, raw_vals)) + else: + other_fields.append((path, raw_vals, cog_degs, sog_kts)) + + # Print COG candidates + print(f"\n*** POTENTIAL COG FIELDS (all {analyzed} samples matched {args.cog_min}°-{args.cog_max}°) ***") + if cog_candidates: + for path, degs, raws in cog_candidates: + min_deg, max_deg = min(degs), max(degs) + print(f" {path}: {min_deg:.1f}° - {max_deg:.1f}° (raw: {min(raws):.4f} - {max(raws):.4f} rad)") + else: + print(" (none found)") + + # Print SOG candidates + print(f"\n*** POTENTIAL SOG FIELDS (all {analyzed} samples matched {args.sog_min}-{args.sog_max} kts) ***") + if sog_candidates: + for path, kts_list, raws in sog_candidates: + min_kts, max_kts = min(kts_list), max(kts_list) + print(f" {path}: {min_kts:.2f} - {max_kts:.2f} kts (raw: {min(raws):.4f} - {max(raws):.4f} m/s)") + else: + print(" (none found)") + + # Print other navigation-looking fields (small positive values) + print(f"\n*** OTHER NUMERIC FIELDS (may be COG/SOG with different interpretation) ***") + nav_fields = [(p, r, c, s) for p, r, c, s in other_fields + if len(r) > 0 and 0 < min(r) < 100 and max(r) < 1000] + + for path, raws, cog_degs, sog_kts in sorted(nav_fields, key=lambda x: x[0]): + min_raw, max_raw = min(raws), max(raws) + info = f" {path}: raw {min_raw:.4f} - {max_raw:.4f}" + if cog_degs: + info += f" | as deg: {min(cog_degs):.1f}° - {max(cog_degs):.1f}°" + if sog_kts: + info += f" | as kts: {min(sog_kts):.2f} - {max(sog_kts):.2f}" + print(info) + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/debug/find_consistent_heading.py b/axiom-nmea/debug/find_consistent_heading.py new file mode 100644 index 0000000..f780f55 --- /dev/null +++ b/axiom-nmea/debug/find_consistent_heading.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Find offsets that show consistent heading-like values in BOTH pcaps.""" + +import struct +from collections import defaultdict + +def decode_float(data, offset): + if offset + 4 > len(data): + return None + try: + val = struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + +# Load both pcaps +pcap1 = read_pcap("raymarine_sample.pcap") +pcap2 = read_pcap("raymarine_sample_twd_69-73.pcap") + +print("Looking for offsets where values differ significantly between captures") +print("(indicating actual sensor data that changed, like heading)\n") + +# Focus on 344 and 446 byte packets (most common with wind data) +for target_size in [344, 446]: + print(f"\n{'='*70}") + print(f"PACKET SIZE: {target_size} bytes") + print("=" * 70) + + pkts1 = [p for p in pcap1 if len(p) == target_size][:10] + pkts2 = [p for p in pcap2 if len(p) == target_size][:10] + + if not pkts1 or not pkts2: + print(" Not enough packets") + continue + + print(f"\n{'Offset':<10} {'Original pcap':<20} {'TWD pcap':<20} {'Diff':<10}") + print("-" * 60) + + # Check offsets from 0x50 to 0x180 + for offset in range(0x50, min(target_size - 4, 0x180)): + vals1 = [decode_float(p, offset) for p in pkts1] + vals2 = [decode_float(p, offset) for p in pkts2] + + # Filter valid radian values (0 to 2*pi) + degs1 = [v * 57.2958 for v in vals1 if v and 0 < v < 6.5] + degs2 = [v * 57.2958 for v in vals2 if v and 0 < v < 6.5] + + if not degs1 or not degs2: + continue + + avg1 = sum(degs1) / len(degs1) + avg2 = sum(degs2) / len(degs2) + diff = abs(avg2 - avg1) + + # Look for offsets where: + # 1. Values are valid angles (not 0, not garbage) + # 2. Values changed between captures (diff > 5°) + # 3. Both values are in reasonable heading range (0-360°) + if 5 < avg1 < 355 and 5 < avg2 < 355 and diff > 5: + print(f"0x{offset:04x} {avg1:6.1f}° ({len(degs1)} hits) {avg2:6.1f}° ({len(degs2)} hits) {diff:5.1f}°") diff --git a/axiom-nmea/debug/find_heading_vs_twd.py b/axiom-nmea/debug/find_heading_vs_twd.py new file mode 100644 index 0000000..0108608 --- /dev/null +++ b/axiom-nmea/debug/find_heading_vs_twd.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +""" +Find heading vs TWD candidates. +At anchor pointing into wind: heading and TWD should be within ~50° of each other. +TWD expected: 69-73°, so heading could be ~20-120° +""" + +import struct +from collections import defaultdict + +def decode_float(data, offset): + if offset + 4 > len(data): + return None + try: + val = struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + +# Search for angles in a wider range around expected TWD (69-73°) +# Heading at anchor into wind could be 20-120° +TARGET_DEG_MIN = 15 +TARGET_DEG_MAX = 130 +TARGET_RAD_MIN = TARGET_DEG_MIN * 0.0174533 +TARGET_RAD_MAX = TARGET_DEG_MAX * 0.0174533 + +print("Reading raymarine_sample_twd_69-73.pcap...") +packets = read_pcap("raymarine_sample_twd_69-73.pcap") +print(f"Loaded {len(packets)} packets") +print(f"\nSearching for angles {TARGET_DEG_MIN}-{TARGET_DEG_MAX}° (could be heading or TWD)") +print("Expected: TWD ~69-73°, Heading within ±50° of that\n") + +# Track candidates by offset +candidates = defaultdict(list) + +for pkt_idx, pkt in enumerate(packets): + pkt_len = len(pkt) + if pkt_len < 100: + continue + + for offset in range(0x50, min(pkt_len - 4, 0x200)): + val = decode_float(pkt, offset) + if val is None: + continue + + # Check if in target radian range + if TARGET_RAD_MIN <= val <= TARGET_RAD_MAX: + deg = val * 57.2958 + candidates[offset].append((pkt_idx, pkt_len, val, deg)) + +print("=" * 70) +print("ANGLE CANDIDATES (heading or TWD)") +print("Filtering for offsets with >20 hits in target packet sizes") +print("=" * 70) + +# Focus on packet sizes known to have wind data +target_sizes = {344, 446, 788, 888, 931, 1031, 1472} + +for offset in sorted(candidates.keys()): + hits = candidates[offset] + # Filter to target packet sizes + target_hits = [(i, s, v, d) for i, s, v, d in hits if s in target_sizes] + + if len(target_hits) < 10: + continue + + degs = [d for _, _, _, d in target_hits] + pkt_sizes = sorted(set(s for _, s, _, _ in target_hits)) + + avg_deg = sum(degs) / len(degs) + min_deg = min(degs) + max_deg = max(degs) + + # Categorize based on value + if 65 <= avg_deg <= 80: + category = "*** LIKELY TWD ***" + elif 20 <= avg_deg <= 50 or 90 <= avg_deg <= 120: + category = " (could be heading)" + else: + category = "" + + print(f"\n 0x{offset:04x}: {len(target_hits):3d} hits, avg={avg_deg:5.1f}°, range={min_deg:.1f}°-{max_deg:.1f}° {category}") + print(f" Sizes: {pkt_sizes}") + +# Now compare offset 0x006b with nearby offsets +print("\n" + "=" * 70) +print("DETAILED COMPARISON: 0x006b vs nearby offsets") +print("=" * 70) + +check_offsets = [0x0066, 0x006b, 0x0070, 0x0075, 0x007a] + +for pkt_len in [344, 446, 788, 888]: + matching = [p for p in packets if len(p) == pkt_len][:3] + if not matching: + continue + + print(f"\n {pkt_len} byte packets (first 3):") + for off in check_offsets: + vals = [decode_float(p, off) for p in matching] + degs = [f"{v * 57.2958:.1f}°" if v and 0 <= v <= 6.5 else "---" for v in vals] + print(f" 0x{off:04x}: {degs}") diff --git a/axiom-nmea/debug/find_twd.py b/axiom-nmea/debug/find_twd.py new file mode 100644 index 0000000..2c7688b --- /dev/null +++ b/axiom-nmea/debug/find_twd.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Search for True Wind Direction values in the 69-73 degree range.""" + +import struct +from collections import defaultdict + +# 69-73 degrees in radians +TARGET_DEG_MIN = 66 # slightly wider range +TARGET_DEG_MAX = 76 +TARGET_RAD_MIN = TARGET_DEG_MIN * 0.0174533 # ~1.15 rad +TARGET_RAD_MAX = TARGET_DEG_MAX * 0.0174533 # ~1.33 rad + +def decode_float(data, offset): + if offset + 4 > len(data): + return None + try: + val = struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + +print(f"Reading raymarine_sample_twd_69-73.pcap...") +packets = read_pcap("raymarine_sample_twd_69-73.pcap") +print(f"Loaded {len(packets)} packets\n") + +print(f"Searching for direction values {TARGET_DEG_MIN}-{TARGET_DEG_MAX}° ({TARGET_RAD_MIN:.3f}-{TARGET_RAD_MAX:.3f} rad)\n") + +# Track candidates by offset +candidates = defaultdict(list) + +for pkt_idx, pkt in enumerate(packets): + pkt_len = len(pkt) + if pkt_len < 100: + continue + + for offset in range(0x30, min(pkt_len - 4, 0x400)): + val = decode_float(pkt, offset) + if val is None: + continue + + # Check if in target radian range + if TARGET_RAD_MIN <= val <= TARGET_RAD_MAX: + deg = val * 57.2958 + candidates[offset].append((pkt_idx, pkt_len, val, deg)) + +print("=" * 70) +print("WIND DIRECTION CANDIDATES (by offset, sorted by hit count)") +print("=" * 70) + +# Sort by number of hits +sorted_offsets = sorted(candidates.keys(), key=lambda x: -len(candidates[x])) + +for offset in sorted_offsets[:25]: + hits = candidates[offset] + values = [v for _, _, v, _ in hits] + degs = [d for _, _, _, d in hits] + pkt_sizes = sorted(set(s for _, s, _, _ in hits)) + + avg_rad = sum(values) / len(values) + avg_deg = sum(degs) / len(degs) + min_deg = min(degs) + max_deg = max(degs) + + print(f"\n Offset 0x{offset:04x}: {len(hits):4d} hits") + print(f" Degrees: avg={avg_deg:.1f}°, range={min_deg:.1f}°-{max_deg:.1f}°") + print(f" Radians: avg={avg_rad:.4f}") + print(f" Packet sizes: {pkt_sizes[:8]}{'...' if len(pkt_sizes) > 8 else ''}") diff --git a/axiom-nmea/debug/find_twd_hdg.py b/axiom-nmea/debug/find_twd_hdg.py new file mode 100644 index 0000000..a731748 --- /dev/null +++ b/axiom-nmea/debug/find_twd_hdg.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +Find TWD and Heading offsets using known values: +- TWD: 62-70° +- Heading: 29-35° +""" + +import struct +from collections import defaultdict + +# Known ranges +TWD_DEG_MIN, TWD_DEG_MAX = 60, 72 # Slightly wider +HDG_DEG_MIN, HDG_DEG_MAX = 27, 37 # Slightly wider + +TWD_RAD_MIN = TWD_DEG_MIN * 0.0174533 +TWD_RAD_MAX = TWD_DEG_MAX * 0.0174533 +HDG_RAD_MIN = HDG_DEG_MIN * 0.0174533 +HDG_RAD_MAX = HDG_DEG_MAX * 0.0174533 + +def decode_float(data, offset): + if offset + 4 > len(data): + return None + try: + val = struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + +print("Reading raymarine_sample_TWD_62-70_HDG_29-35.pcap...") +packets = read_pcap("raymarine_sample_TWD_62-70_HDG_29-35.pcap") +print(f"Loaded {len(packets)} packets\n") + +print(f"Searching for:") +print(f" TWD: {TWD_DEG_MIN}-{TWD_DEG_MAX}° ({TWD_RAD_MIN:.3f}-{TWD_RAD_MAX:.3f} rad)") +print(f" HDG: {HDG_DEG_MIN}-{HDG_DEG_MAX}° ({HDG_RAD_MIN:.3f}-{HDG_RAD_MAX:.3f} rad)\n") + +# Track candidates +twd_candidates = defaultdict(list) +hdg_candidates = defaultdict(list) + +target_sizes = {344, 446, 788, 888, 931, 1031, 1472} + +for pkt_idx, pkt in enumerate(packets): + pkt_len = len(pkt) + if pkt_len not in target_sizes: + continue + + for offset in range(0x50, min(pkt_len - 4, 0x200)): + val = decode_float(pkt, offset) + if val is None: + continue + + deg = val * 57.2958 + + if TWD_RAD_MIN <= val <= TWD_RAD_MAX: + twd_candidates[offset].append((pkt_len, deg)) + + if HDG_RAD_MIN <= val <= HDG_RAD_MAX: + hdg_candidates[offset].append((pkt_len, deg)) + +print("=" * 70) +print(f"TWD CANDIDATES ({TWD_DEG_MIN}-{TWD_DEG_MAX}°)") +print("=" * 70) + +for offset in sorted(twd_candidates.keys(), key=lambda x: -len(twd_candidates[x]))[:15]: + hits = twd_candidates[offset] + degs = [d for _, d in hits] + sizes = sorted(set(s for s, _ in hits)) + avg = sum(degs) / len(degs) + print(f" 0x{offset:04x}: {len(hits):3d} hits, avg={avg:.1f}°, range={min(degs):.1f}-{max(degs):.1f}°, sizes={sizes}") + +print("\n" + "=" * 70) +print(f"HEADING CANDIDATES ({HDG_DEG_MIN}-{HDG_DEG_MAX}°)") +print("=" * 70) + +for offset in sorted(hdg_candidates.keys(), key=lambda x: -len(hdg_candidates[x]))[:15]: + hits = hdg_candidates[offset] + degs = [d for _, d in hits] + sizes = sorted(set(s for s, _ in hits)) + avg = sum(degs) / len(degs) + print(f" 0x{offset:04x}: {len(hits):3d} hits, avg={avg:.1f}°, range={min(degs):.1f}-{max(degs):.1f}°, sizes={sizes}") + +# Cross-check: find offsets that appear in both (shouldn't happen if ranges don't overlap) +common = set(twd_candidates.keys()) & set(hdg_candidates.keys()) +if common: + print(f"\n⚠️ Offsets appearing in both ranges: {[hex(o) for o in common]}") diff --git a/axiom-nmea/debug/find_twd_precise.py b/axiom-nmea/debug/find_twd_precise.py new file mode 100644 index 0000000..8fbe5de --- /dev/null +++ b/axiom-nmea/debug/find_twd_precise.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Find all float values in 69-73 degree range more precisely.""" + +import struct +from collections import defaultdict + +# Exact expected range: 69-73 degrees +TARGET_DEG_MIN = 68 +TARGET_DEG_MAX = 74 +TARGET_RAD_MIN = TARGET_DEG_MIN * 0.0174533 +TARGET_RAD_MAX = TARGET_DEG_MAX * 0.0174533 + +def decode_float(data, offset): + if offset + 4 > len(data): + return None + try: + val = struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + +print(f"Reading raymarine_sample_twd_69-73.pcap...") +packets = read_pcap("raymarine_sample_twd_69-73.pcap") +print(f"Loaded {len(packets)} packets\n") + +print(f"Searching PRECISELY for {TARGET_DEG_MIN}-{TARGET_DEG_MAX}° ({TARGET_RAD_MIN:.4f}-{TARGET_RAD_MAX:.4f} rad)\n") + +# Track candidates by offset +candidates = defaultdict(list) + +for pkt_idx, pkt in enumerate(packets): + pkt_len = len(pkt) + if pkt_len < 100: + continue + + for offset in range(0x30, min(pkt_len - 4, 0x500)): + val = decode_float(pkt, offset) + if val is None: + continue + + # Check if in target radian range + if TARGET_RAD_MIN <= val <= TARGET_RAD_MAX: + deg = val * 57.2958 + candidates[offset].append((pkt_idx, pkt_len, val, deg)) + +print("=" * 70) +print(f"OFFSETS WITH VALUES IN {TARGET_DEG_MIN}-{TARGET_DEG_MAX}° RANGE") +print("=" * 70) + +# Sort by number of hits +sorted_offsets = sorted(candidates.keys(), key=lambda x: -len(candidates[x])) + +for offset in sorted_offsets[:30]: + hits = candidates[offset] + values = [v for _, _, v, _ in hits] + degs = [d for _, _, _, d in hits] + pkt_sizes = sorted(set(s for _, s, _, _ in hits)) + + avg_deg = sum(degs) / len(degs) + min_deg = min(degs) + max_deg = max(degs) + + print(f"\n 0x{offset:04x}: {len(hits):3d} hits, avg={avg_deg:.1f}°, range={min_deg:.1f}°-{max_deg:.1f}°") + print(f" Packet sizes: {pkt_sizes}") + +# Also show what's at offset 0x0070 since it had good results +print("\n" + "=" * 70) +print("DETAILED CHECK OF OFFSET 0x0070 (from earlier analysis)") +print("=" * 70) + +for pkt_len in [344, 446, 788, 888, 931, 1031, 1472]: + matching = [p for p in packets if len(p) == pkt_len][:5] + if matching: + vals = [decode_float(p, 0x0070) for p in matching] + degs = [v * 57.2958 if v else None for v in vals] + print(f" {pkt_len} bytes: {[f'{d:.1f}°' if d else 'N/A' for d in degs]}") diff --git a/axiom-nmea/debug/packet_debug.py b/axiom-nmea/debug/packet_debug.py new file mode 100755 index 0000000..447db56 --- /dev/null +++ b/axiom-nmea/debug/packet_debug.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +""" +Raymarine Packet Debug Tool + +Dumps raw protobuf field structure from Raymarine multicast packets. +Use this to discover field locations for COG, SOG, and other data. + +Usage: + python packet_debug.py -i 198.18.5.5 + +The tool shows: +- All top-level protobuf fields and their wire types +- Nested field structures with decoded values +- Float/double interpretations for potential navigation data +""" + +import argparse +import logging +import os +import signal +import socket +import struct +import sys +import time +from datetime import datetime +from typing import Dict, Any, Optional, List + +# Add parent directory to path for library import +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from raymarine_nmea.protocol.parser import ProtobufParser, ProtoField +from raymarine_nmea.protocol.constants import ( + WIRE_VARINT, WIRE_FIXED64, WIRE_LENGTH, WIRE_FIXED32, + HEADER_SIZE, RAD_TO_DEG, MS_TO_KTS, +) +from raymarine_nmea.sensors import MULTICAST_GROUPS + +# Known field names for reference +FIELD_NAMES = { + 1: "DEVICE_INFO", + 2: "GPS_POSITION", + 3: "HEADING", + 7: "DEPTH", + 13: "WIND_NAVIGATION", + 14: "ENGINE_DATA", + 15: "TEMPERATURE", + 16: "TANK_DATA", + 20: "HOUSE_BATTERY", +} + +WIRE_TYPE_NAMES = { + 0: "varint", + 1: "fixed64", + 2: "length", + 5: "fixed32", +} + +running = True + + +def signal_handler(signum, frame): + global running + running = False + + +def decode_as_float(raw: bytes) -> Optional[float]: + """Try to decode bytes as float.""" + if len(raw) == 4: + try: + val = struct.unpack(' Optional[float]: + """Try to decode bytes as double.""" + if len(raw) == 8: + try: + val = struct.unpack(' List[str]: + """Format a protobuf field for display.""" + lines = [] + prefix = " " * indent + wire_name = WIRE_TYPE_NAMES.get(pf.wire_type, f"wire{pf.wire_type}") + field_name = FIELD_NAMES.get(pf.field_num, "") + if field_name: + field_name = f" ({field_name})" + + if pf.wire_type == WIRE_VARINT: + lines.append(f"{prefix}Field {pf.field_num}{field_name}: {pf.value} [{wire_name}]") + + elif pf.wire_type == WIRE_FIXED32: + fval = decode_as_float(pf.value) + hex_str = pf.value.hex() + if fval is not None: + # Show various interpretations + deg = fval * RAD_TO_DEG if 0 <= fval <= 6.5 else None + kts = fval * MS_TO_KTS if 0 <= fval <= 50 else None + interp = [] + if deg is not None and 0 <= deg <= 360: + interp.append(f"{deg:.1f}°") + if kts is not None and 0 <= kts <= 100: + interp.append(f"{kts:.1f}kts") + interp_str = f" -> {', '.join(interp)}" if interp else "" + lines.append(f"{prefix}Field {pf.field_num}{field_name}: {fval:.6f} [{wire_name}] 0x{hex_str}{interp_str}") + else: + lines.append(f"{prefix}Field {pf.field_num}{field_name}: 0x{hex_str} [{wire_name}]") + + elif pf.wire_type == WIRE_FIXED64: + dval = decode_as_double(pf.value) + hex_str = pf.value.hex() + if dval is not None: + lines.append(f"{prefix}Field {pf.field_num}{field_name}: {dval:.8f} [{wire_name}]") + else: + lines.append(f"{prefix}Field {pf.field_num}{field_name}: 0x{hex_str} [{wire_name}]") + + elif pf.wire_type == WIRE_LENGTH: + data_len = len(pf.value) if isinstance(pf.value, bytes) else 0 + lines.append(f"{prefix}Field {pf.field_num}{field_name}: [{wire_name}, {data_len} bytes]") + if pf.children: + for child_num in sorted(pf.children.keys()): + child = pf.children[child_num] + lines.extend(format_value(child, indent + 1)) + elif data_len <= 32: + lines.append(f"{prefix} Raw: 0x{pf.value.hex()}") + + return lines + + +def dump_packet(packet: bytes, packet_num: int, filter_fields: Optional[set] = None): + """Dump a single packet's protobuf structure.""" + if len(packet) < HEADER_SIZE + 10: + return + + proto_data = packet[HEADER_SIZE:] + parser = ProtobufParser(proto_data) + + # Collect repeated fields that we know about + fields = parser.parse_message(collect_repeated={14, 16, 20}) + + if not fields: + return + + # Filter if requested + if filter_fields: + fields = {k: v for k, v in fields.items() if k in filter_fields} + if not fields: + return + + timestamp = datetime.now().strftime('%H:%M:%S.%f')[:-3] + print(f"\n{'='*70}") + print(f"Packet #{packet_num} at {timestamp} ({len(packet)} bytes)") + print(f"{'='*70}") + + for field_num in sorted(fields.keys()): + val = fields[field_num] + # Handle repeated fields (stored as list) + if isinstance(val, list): + for i, pf in enumerate(val): + lines = format_value(pf) + for line in lines: + # Add index to first line for repeated fields + if lines.index(line) == 0: + print(f"{line} [{i}]") + else: + print(line) + else: + for line in format_value(val): + print(line) + + +def main(): + global running + + parser = argparse.ArgumentParser(description="Debug Raymarine packet structure") + parser.add_argument('-i', '--interface', required=True, + help='Interface IP for Raymarine multicast (e.g., 198.18.5.5)') + parser.add_argument('-n', '--count', type=int, default=0, + help='Number of packets to capture (0 = unlimited)') + parser.add_argument('-f', '--fields', type=str, default='', + help='Comma-separated field numbers to show (empty = all)') + parser.add_argument('--interval', type=float, default=0.0, + help='Minimum interval between packets shown (seconds)') + parser.add_argument('--nav-only', action='store_true', + help='Show only navigation-related fields (2,3,13)') + + args = parser.parse_args() + + # Parse field filter + filter_fields = None + if args.nav_only: + filter_fields = {2, 3, 13} # GPS, Heading, Wind/Nav + elif args.fields: + try: + filter_fields = set(int(x.strip()) for x in args.fields.split(',')) + except ValueError: + print("Error: --fields must be comma-separated integers") + sys.exit(1) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # Create sockets for all multicast groups + sockets = [] + for group, port in MULTICAST_GROUPS: + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, 'SO_REUSEPORT'): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind(('', port)) + + # Join multicast group using struct.pack like MulticastListener does + mreq = struct.pack("4s4s", + socket.inet_aton(group), + socket.inet_aton(args.interface) + ) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + sock.setblocking(False) + sockets.append((sock, group, port)) + print(f"Joined {group}:{port}") + except Exception as e: + print(f"Error joining {group}:{port}: {e}") + + if not sockets: + print("Error: Could not join any multicast groups") + sys.exit(1) + + print(f"\nRaymarine Packet Debug Tool") + print(f"Listening on {args.interface}") + if filter_fields: + print(f"Showing fields: {sorted(filter_fields)}") + print(f"Press Ctrl+C to stop\n") + + print("Known field numbers:") + for num, name in sorted(FIELD_NAMES.items()): + print(f" {num}: {name}") + print() + + packet_count = 0 + last_dump = 0 + + try: + while running: + # Poll all sockets + for sock, group, port in sockets: + try: + data, addr = sock.recvfrom(65535) + packet_count += 1 + + # Rate limiting + now = time.time() + if args.interval > 0 and (now - last_dump) < args.interval: + continue + + dump_packet(data, packet_count, filter_fields) + last_dump = now + + # Count limit + if args.count > 0 and packet_count >= args.count: + running = False + break + + except BlockingIOError: + continue + except Exception as e: + if running: + print(f"Error on {group}:{port}: {e}") + + time.sleep(0.01) # Small sleep to avoid busy-waiting + + finally: + for sock, _, _ in sockets: + sock.close() + print(f"\n\nCaptured {packet_count} packets") + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/debug/pressure_finder.py b/axiom-nmea/debug/pressure_finder.py new file mode 100755 index 0000000..c1fe8c2 --- /dev/null +++ b/axiom-nmea/debug/pressure_finder.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +""" +Pressure Finder - Locate barometric pressure data in Raymarine protobuf stream. + +Scans for float values that could be pressure in various units. +Uses known pressure value to correlate field locations. + +Usage: + python pressure_finder.py -i YOUR_INTERFACE_IP -p 1021 # Known pressure in mbar +""" + +import struct +import socket +import time +import argparse +from typing import Dict, List, Any, Optional +from collections import defaultdict + +WIRE_VARINT = 0 +WIRE_FIXED64 = 1 +WIRE_LENGTH = 2 +WIRE_FIXED32 = 5 + +HEADER_SIZE = 20 + +MULTICAST_GROUPS = [ + ("226.192.206.102", 2565), # Main sensor data + ("239.2.1.1", 2154), # May contain additional data +] + + +class ProtobufParser: + """Parse protobuf without schema.""" + + def __init__(self, data: bytes): + self.data = data + self.pos = 0 + + def remaining(self): + return len(self.data) - self.pos + + def read_varint(self) -> int: + result = 0 + shift = 0 + while self.pos < len(self.data): + byte = self.data[self.pos] + self.pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + return result + + def parse_nested_deep(self, data: bytes, path: str = "", depth: int = 0, max_depth: int = 5) -> List[tuple]: + """Recursively parse nested protobuf and return list of (path, type, value) tuples.""" + results = [] + pos = 0 + + if depth > max_depth: + return results + + while pos < len(data): + if pos >= len(data): + break + try: + # Read tag + tag_byte = data[pos] + pos += 1 + + # Handle multi-byte varints for tag + tag = tag_byte & 0x7F + shift = 7 + while tag_byte & 0x80 and pos < len(data): + tag_byte = data[pos] + pos += 1 + tag |= (tag_byte & 0x7F) << shift + shift += 7 + + field_num = tag >> 3 + wire_type = tag & 0x07 + + if field_num == 0 or field_num > 100: + break + + field_path = f"{path}.{field_num}" if path else str(field_num) + + if wire_type == WIRE_VARINT: + val = 0 + shift = 0 + while pos < len(data): + byte = data[pos] + pos += 1 + val |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + results.append((field_path, 'varint', val)) + + elif wire_type == WIRE_FIXED32: + raw = data[pos:pos + 4] + pos += 4 + try: + f = struct.unpack('= 2: + nested_results = self.parse_nested_deep(raw, field_path, depth + 1, max_depth) + if nested_results: + results.extend(nested_results) + else: + break + + except Exception: + break + + return results + + +def is_pressure_like(val: float, target_mbar: float) -> Optional[str]: + """Check if a float value could be barometric pressure. + + Checks multiple unit possibilities and returns match description. + """ + tolerance = 0.02 # 2% tolerance + + # Convert target to various units + target_pa = target_mbar * 100 # Pascals (1021 mbar = 102100 Pa) + target_hpa = target_mbar # hPa = mbar + target_kpa = target_mbar / 10 # kPa (102.1) + target_bar = target_mbar / 1000 # bar (1.021) + target_inhg = target_mbar * 0.02953 # inHg (~30.15) + target_psi = target_mbar * 0.0145 # PSI (~14.8) + + checks = [ + (target_mbar, "mbar (direct)"), + (target_hpa, "hPa"), + (target_pa, "Pascals"), + (target_kpa, "kPa"), + (target_bar, "bar"), + (target_inhg, "inHg"), + (target_psi, "PSI"), + ] + + for target, unit in checks: + if target > 0 and abs(val - target) / target < tolerance: + return unit + + return None + + +def scan_packet(data: bytes, target_mbar: float, group: str, port: int, + candidates: Dict[str, Dict]) -> None: + """Scan packet for pressure-like values.""" + if len(data) < HEADER_SIZE + 5: + return + + proto_data = data[HEADER_SIZE:] + parser = ProtobufParser(proto_data) + + # Get all fields recursively + all_fields = parser.parse_all_fields() if hasattr(parser, 'parse_all_fields') else {} + + # Deep parse all length-delimited fields + parser = ProtobufParser(proto_data) + parser.pos = 0 + all_results = parser.parse_nested_deep(proto_data, "") + + for path, vtype, value in all_results: + if vtype in ('float', 'double'): + unit = is_pressure_like(value, target_mbar) + if unit: + if path not in candidates: + candidates[path] = { + 'values': [], + 'unit': unit, + 'type': vtype, + 'count': 0 + } + candidates[path]['values'].append(value) + candidates[path]['count'] += 1 + + +def parse_all_fields(self) -> Dict[int, List[Any]]: + """Parse and collect all fields.""" + fields = {} + + while self.pos < len(self.data): + if self.remaining() < 1: + break + try: + tag = self.read_varint() + field_num = tag >> 3 + wire_type = tag & 0x07 + + if field_num == 0 or field_num > 1000: + break + + if wire_type == WIRE_VARINT: + value = ('varint', self.read_varint()) + elif wire_type == WIRE_FIXED64: + raw = self.data[self.pos:self.pos + 8] + self.pos += 8 + try: + d = struct.unpack(' info + packet_count = 0 + end_time = time.time() + args.time + + try: + while time.time() < end_time: + try: + data, _ = sock.recvfrom(65535) + packet_count += 1 + scan_packet(data, args.pressure, args.group, args.port, candidates) + except socket.timeout: + continue + except KeyboardInterrupt: + pass + finally: + sock.close() + + print(f"\n\nResults after scanning {packet_count} packets:") + print("=" * 60) + + if not candidates: + print("No pressure-like values found!") + print("\nSuggestions:") + print("1. Verify the pressure sensor is connected and broadcasting") + print("2. Try different multicast groups (239.2.1.1:2154)") + print("3. Check if pressure is in a different packet size") + else: + print(f"\nFound {len(candidates)} candidate field(s):\n") + + # Sort by count (most frequent first) + for path, info in sorted(candidates.items(), key=lambda x: -x[1]['count']): + values = info['values'] + avg_val = sum(values) / len(values) + min_val = min(values) + max_val = max(values) + + print(f"Field {path}:") + print(f" Type: {info['type']}") + print(f" Matches: {info['count']} packets") + print(f" Unit likely: {info['unit']}") + print(f" Values: min={min_val:.2f}, max={max_val:.2f}, avg={avg_val:.2f}") + if info['unit'] == 'Pascals': + print(f" As mbar: {avg_val/100:.1f} mbar") + elif info['unit'] == 'kPa': + print(f" As mbar: {avg_val*10:.1f} mbar") + print() + + print("\nDone.") + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/debug/protobuf_decoder.py b/axiom-nmea/debug/protobuf_decoder.py new file mode 100644 index 0000000..d21d398 --- /dev/null +++ b/axiom-nmea/debug/protobuf_decoder.py @@ -0,0 +1,1042 @@ +#!/usr/bin/env python3 +""" +Raymarine Protobuf Decoder + +Properly parses the protobuf structure by field numbers rather than fixed byte offsets. +This is more robust and handles different packet sizes correctly. + +Field Mapping (discovered through analysis): + Field 2: GPS/Position Data + ├─ Field 1 (double): Latitude + └─ Field 2 (double): Longitude + + Field 3: Heading Block + └─ Field 2 (float): Heading (radians) + + Field 5: SOG/COG Navigation Data + ├─ Field 1 (float): COG - Course Over Ground (radians) + └─ Field 3 (float): SOG - Speed Over Ground (m/s) + + Field 7: Depth Block (only in larger packets 1472B+) + └─ Field 1 (float): Depth (meters) + + Field 13: Wind/Navigation Data + ├─ Field 4 (float): True Wind Direction (radians) + ├─ Field 5 (float): Wind Speed (m/s) + └─ Field 6 (float): Apparent Wind Speed (m/s) + + Field 14: Engine Data (repeated) + ├─ Field 1 (varint): Engine ID (0=Port, 1=Starboard) + └─ Field 3: Engine Sensor Data + └─ Field 4 (float): Battery Voltage (volts) + + Field 15: Temperature Data + ├─ Field 3 (float): Air Temperature (Kelvin) + └─ Field 9 (float): Water Temperature (Kelvin) + + Field 16: Tank Data (repeated) + ├─ Field 1 (varint): Tank ID + ├─ Field 2 (varint): Status/Flag + └─ Field 3 (float): Tank Level (percentage) + + Field 20: House Battery Data (repeated) + ├─ Field 1 (varint): Battery ID (11=Aft House, 13=Stern House) + └─ Field 3 (float): Voltage (volts) +""" + +import struct +import socket +import time +import json +import threading +import argparse +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Any, Tuple +from datetime import datetime + +# Wire types +WIRE_VARINT = 0 +WIRE_FIXED64 = 1 +WIRE_LENGTH = 2 +WIRE_FIXED32 = 5 + +# Conversion constants +RAD_TO_DEG = 57.2957795131 +MS_TO_KTS = 1.94384449 +FEET_TO_M = 0.3048 +KELVIN_OFFSET = 273.15 + +# Tank ID to name mapping (id -> (name, capacity_gallons)) +TANK_NAMES = { + 1: ("Stbd Fuel", 265), # Starboard fuel tank, 265 gallons + 2: ("Port Fuel", 265), # Port fuel tank, 265 gallons + 10: ("Fwd Water", 90), # Forward water tank, 90 gallons + 11: ("Aft Water", 90), # Rear water tank, 90 gallons + # Inferred tanks (status-based) + 100: ("Black Water", 53), # Waste tank (status=5), 53 gallons +} + +# Tank status values +TANK_STATUS_WASTE = 5 # Black/gray water tanks use status=5 + +# Battery ID to name mapping (id -> name) +# House batteries from Field 20: IDs 11, 13 +# Engine batteries from Field 14: IDs 1000+ (1000 + engine_id) +BATTERY_NAMES = { + 11: "Aft House", # Aft house battery bank (Field 20) + 13: "Stern House", # Stern house battery bank (Field 20) + 1000: "Port Engine", # Port engine battery (Field 14, engine_id=0) + 1001: "Stbd Engine", # Starboard engine battery (Field 14, engine_id=1) +} + +# Raymarine multicast groups - ALL groups (original) +MULTICAST_GROUPS_ALL = [ + ("226.192.206.98", 2561), # Navigation sensors (mostly zeros) + ("226.192.206.99", 2562), # Heartbeat/status + ("226.192.206.100", 2563), # Alternative data (low traffic) + ("226.192.206.101", 2564), # Alternative data (low traffic) + ("226.192.206.102", 2565), # PRIMARY sensor data + ("226.192.219.0", 3221), # Display sync (high traffic, no sensor data) + ("239.2.1.1", 2154), # May contain tank/engine data +] + +# PRIMARY only - single group with all sensor data +MULTICAST_GROUPS_PRIMARY = [ + ("226.192.206.102", 2565), # PRIMARY sensor data (GPS, wind, depth, tanks, batteries) +] + +# MINIMAL - primary + potential tank/engine backup +MULTICAST_GROUPS_MINIMAL = [ + ("226.192.206.102", 2565), # PRIMARY sensor data + ("239.2.1.1", 2154), # Tank/engine data (backup) +] + +# Group presets for CLI +GROUP_PRESETS = { + "all": MULTICAST_GROUPS_ALL, + "primary": MULTICAST_GROUPS_PRIMARY, + "minimal": MULTICAST_GROUPS_MINIMAL, +} + +# Default to all groups for backward compatibility +MULTICAST_GROUPS = MULTICAST_GROUPS_ALL + +# Fixed header size +HEADER_SIZE = 20 + + +@dataclass +class ProtoField: + """A decoded protobuf field.""" + field_num: int + wire_type: int + value: Any + children: Dict[int, 'ProtoField'] = field(default_factory=dict) + + +class ProtobufParser: + """Parses protobuf wire format without a schema.""" + + def __init__(self, data: bytes): + self.data = data + self.pos = 0 + + def remaining(self) -> int: + return len(self.data) - self.pos + + def read_varint(self) -> int: + """Decode a varint.""" + result = 0 + shift = 0 + while self.pos < len(self.data): + byte = self.data[self.pos] + self.pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + if shift > 63: + break + return result + + def read_fixed64(self) -> bytes: + """Read 8 bytes.""" + value = self.data[self.pos:self.pos + 8] + self.pos += 8 + return value + + def read_fixed32(self) -> bytes: + """Read 4 bytes.""" + value = self.data[self.pos:self.pos + 4] + self.pos += 4 + return value + + def read_length_delimited(self) -> bytes: + """Read length-prefixed bytes.""" + length = self.read_varint() + value = self.data[self.pos:self.pos + length] + self.pos += length + return value + + def parse_message(self, collect_repeated: set = None) -> Dict[int, Any]: + """Parse all fields in a message, returning dict by field number. + + Args: + collect_repeated: Set of field numbers to collect as lists (for repeated fields) + """ + fields = {} + if collect_repeated is None: + collect_repeated = set() + + while self.pos < len(self.data): + if self.remaining() < 1: + break + + try: + tag = self.read_varint() + field_num = tag >> 3 + wire_type = tag & 0x07 + + if field_num == 0 or field_num > 536870911: + break + + if wire_type == WIRE_VARINT: + value = self.read_varint() + elif wire_type == WIRE_FIXED64: + value = self.read_fixed64() + elif wire_type == WIRE_LENGTH: + value = self.read_length_delimited() + elif wire_type == WIRE_FIXED32: + value = self.read_fixed32() + else: + break # Unknown wire type + + # For length-delimited, try to parse as nested message + children = {} + if wire_type == WIRE_LENGTH and len(value) >= 2: + try: + nested_parser = ProtobufParser(value) + children = nested_parser.parse_message() + # Only keep if we parsed most of the data + if nested_parser.pos < len(value) * 0.5: + children = {} + except: + children = {} + + pf = ProtoField(field_num, wire_type, value, children) + + # Handle repeated fields - collect as list + if field_num in collect_repeated: + if field_num not in fields: + fields[field_num] = [] + fields[field_num].append(pf) + else: + # Keep last occurrence for non-repeated fields + fields[field_num] = pf + + except (IndexError, struct.error): + break + + return fields + + +@dataclass +class SensorData: + """Current sensor readings.""" + # Position + latitude: Optional[float] = None + longitude: Optional[float] = None + + # Navigation + heading_deg: Optional[float] = None + cog_deg: Optional[float] = None + sog_kts: Optional[float] = None + + # Wind + twd_deg: Optional[float] = None # True Wind Direction + tws_kts: Optional[float] = None # True Wind Speed + awa_deg: Optional[float] = None # Apparent Wind Angle + aws_kts: Optional[float] = None # Apparent Wind Speed + + # Depth + depth_ft: Optional[float] = None + + # Temperature + water_temp_c: Optional[float] = None + air_temp_c: Optional[float] = None + + # Tanks: dict of tank_id -> level percentage + tanks: Dict[int, float] = field(default_factory=dict) + + # Batteries: dict of battery_id -> voltage + batteries: Dict[int, float] = field(default_factory=dict) + + # Timestamps + gps_time: float = 0 + heading_time: float = 0 + wind_time: float = 0 + depth_time: float = 0 + tank_time: float = 0 + battery_time: float = 0 + + # Stats + packet_count: int = 0 + decode_count: int = 0 + start_time: float = field(default_factory=time.time) + + # Per-group statistics: group -> {packets, decoded, last_decode_fields} + group_stats: Dict[str, Dict[str, Any]] = field(default_factory=dict) + + # Thread safety + _lock: threading.Lock = field(default_factory=threading.Lock) + + def to_dict(self) -> Dict[str, Any]: + with self._lock: + return { + "timestamp": datetime.now().isoformat(), + "position": { + "latitude": self.latitude, + "longitude": self.longitude, + }, + "navigation": { + "heading_deg": round(self.heading_deg, 1) if self.heading_deg else None, + "cog_deg": round(self.cog_deg, 1) if self.cog_deg else None, + "sog_kts": round(self.sog_kts, 1) if self.sog_kts else None, + }, + "wind": { + "true_direction_deg": round(self.twd_deg, 1) if self.twd_deg else None, + "true_speed_kts": round(self.tws_kts, 1) if self.tws_kts else None, + "apparent_angle_deg": round(self.awa_deg, 1) if self.awa_deg else None, + "apparent_speed_kts": round(self.aws_kts, 1) if self.aws_kts else None, + }, + "depth": { + "feet": round(self.depth_ft, 1) if self.depth_ft else None, + "meters": round(self.depth_ft * FEET_TO_M, 1) if self.depth_ft else None, + }, + "temperature": { + "water_c": round(self.water_temp_c, 1) if self.water_temp_c else None, + "air_c": round(self.air_temp_c, 1) if self.air_temp_c else None, + }, + "tanks": { + str(tank_id): { + "name": TANK_NAMES.get(tank_id, (f"Tank#{tank_id}", None))[0], + "level_pct": round(level, 1), + "capacity_gal": TANK_NAMES.get(tank_id, (None, None))[1], + } for tank_id, level in self.tanks.items() + }, + "batteries": { + str(battery_id): { + "name": BATTERY_NAMES.get(battery_id, f"Battery#{battery_id}"), + "voltage_v": round(voltage, 1), + } for battery_id, voltage in self.batteries.items() + }, + "stats": { + "packets": self.packet_count, + "decoded": self.decode_count, + "uptime_s": round(time.time() - self.start_time, 1), + } + } + + +class RaymarineDecoder: + """Decodes Raymarine packets using proper protobuf parsing.""" + + def __init__(self, sensor_data: SensorData, verbose: bool = False): + self.data = sensor_data + self.verbose = verbose + + def decode_double(self, raw: bytes) -> Optional[float]: + """Decode 8 bytes as little-endian double.""" + if len(raw) != 8: + return None + try: + val = struct.unpack(' Optional[float]: + """Decode 4 bytes as little-endian float.""" + if len(raw) != 4: + return None + try: + val = struct.unpack(' bool: + """Decode a single Raymarine packet. + + Args: + packet: Raw packet bytes + group: Source multicast group (e.g., "226.192.206.102:2565") + """ + with self.data._lock: + self.data.packet_count += 1 + # Initialize group stats if needed + if group and group not in self.data.group_stats: + self.data.group_stats[group] = { + "packets": 0, + "decoded": 0, + "fields": set(), # Which data types came from this group + } + if group: + self.data.group_stats[group]["packets"] += 1 + + # Need at least header + some protobuf data + if len(packet) < HEADER_SIZE + 20: + return False + + # Skip fixed header, protobuf starts at offset 0x14 + proto_data = packet[HEADER_SIZE:] + + # Parse protobuf - collect repeated fields: + # Field 14 = Engine data (contains battery voltage at 14.3.4) + # Field 16 = Tank data + # Field 20 = House battery data + parser = ProtobufParser(proto_data) + fields = parser.parse_message(collect_repeated={14, 16, 20}) + + if not fields: + return False + + decoded = False + + # Track which fields were decoded from this packet + decoded_fields = [] + + # Extract GPS from Field 2 + if 2 in fields: + gps_field = fields[2] + if gps_field.children: + if self._extract_gps(gps_field.children): + decoded = True + decoded_fields.append("gps") + + # Extract Heading from Field 3 + if 3 in fields: + heading_field = fields[3] + if heading_field.children: + if self._extract_heading(heading_field.children): + decoded = True + decoded_fields.append("heading") + + # Extract COG/SOG from Field 5 + if 5 in fields: + cog_field = fields[5] + if cog_field.children: + if self._extract_cog_sog(cog_field.children): + decoded = True + decoded_fields.append("cog_sog") + + # Extract Wind from Field 13 + if 13 in fields: + wind_field = fields[13] + if wind_field.children: + if self._extract_wind(wind_field.children): + decoded = True + decoded_fields.append("wind") + + # Extract Depth from Field 7 (only in larger packets) + if 7 in fields: + depth_field = fields[7] + if depth_field.children: + if self._extract_depth(depth_field.children): + decoded = True + decoded_fields.append("depth") + + # Extract Temperature from Field 15 + if 15 in fields: + temp_field = fields[15] + if temp_field.children: + if self._extract_temperature(temp_field.children): + decoded = True + decoded_fields.append("temp") + + # Extract Tank data from Field 16 (repeated) + if 16 in fields: + tank_fields = fields[16] # This is a list + if self._extract_tanks(tank_fields): + decoded = True + decoded_fields.append("tanks") + + # Extract Battery data from Field 20 (repeated) - house batteries + if 20 in fields: + battery_fields = fields[20] # This is a list + if self._extract_batteries(battery_fields): + decoded = True + decoded_fields.append("house_batt") + + # Extract Engine battery data from Field 14 (repeated) + if 14 in fields: + engine_fields = fields[14] # This is a list + if self._extract_engine_batteries(engine_fields): + decoded = True + decoded_fields.append("engine_batt") + + if decoded: + with self.data._lock: + self.data.decode_count += 1 + # Track which fields came from which group + if group and group in self.data.group_stats: + self.data.group_stats[group]["decoded"] += 1 + self.data.group_stats[group]["fields"].update(decoded_fields) + + return decoded + + def _extract_gps(self, fields: Dict[int, ProtoField]) -> bool: + """Extract GPS position from Field 2's children.""" + lat = None + lon = None + + # Field 1 = Latitude + if 1 in fields and fields[1].wire_type == WIRE_FIXED64: + lat = self.decode_double(fields[1].value) + + # Field 2 = Longitude + if 2 in fields and fields[2].wire_type == WIRE_FIXED64: + lon = self.decode_double(fields[2].value) + + # Validate lat/lon + if lat is not None and lon is not None: + if -90 <= lat <= 90 and -180 <= lon <= 180: + if abs(lat) > 0.1 or abs(lon) > 0.1: # Not at null island + with self.data._lock: + self.data.latitude = lat + self.data.longitude = lon + self.data.gps_time = time.time() + if self.verbose: + print(f"GPS: {lat:.6f}, {lon:.6f}") + return True + return False + + def _extract_heading(self, fields: Dict[int, ProtoField]) -> bool: + """Extract heading from Field 3's children.""" + # Field 2 = Heading in radians + if 2 in fields and fields[2].wire_type == WIRE_FIXED32: + val = self.decode_float(fields[2].value) + if val is not None and 0 <= val <= 6.5: + heading_deg = (val * RAD_TO_DEG) % 360 + with self.data._lock: + self.data.heading_deg = heading_deg + self.data.heading_time = time.time() + if self.verbose: + print(f"Heading: {heading_deg:.1f}°") + return True + return False + + def _extract_cog_sog(self, fields: Dict[int, ProtoField]) -> bool: + """Extract COG and SOG from Field 5's children.""" + decoded = False + + # Field 1 = COG (Course Over Ground) in radians - confirmed with real data + if 1 in fields and fields[1].wire_type == WIRE_FIXED32: + val = self.decode_float(fields[1].value) + if val is not None and 0 <= val <= 6.5: + cog_deg = (val * RAD_TO_DEG) % 360 + with self.data._lock: + self.data.cog_deg = cog_deg + if self.verbose: + print(f"COG: {cog_deg:.1f}°") + decoded = True + + # Field 3 = SOG (Speed Over Ground) in m/s - confirmed with real data + if 3 in fields and fields[3].wire_type == WIRE_FIXED32: + val = self.decode_float(fields[3].value) + if val is not None and 0 <= val <= 100: + sog_kts = val * MS_TO_KTS + with self.data._lock: + self.data.sog_kts = sog_kts + if self.verbose: + print(f"SOG: {sog_kts:.1f} kts") + decoded = True + + return decoded + + def _extract_wind(self, fields: Dict[int, ProtoField]) -> bool: + """Extract wind data from Field 13's children.""" + decoded = False + + # Field 4 = True Wind Direction (radians) + if 4 in fields and fields[4].wire_type == WIRE_FIXED32: + val = self.decode_float(fields[4].value) + if val is not None and 0 <= val <= 6.5: + twd_deg = (val * RAD_TO_DEG) % 360 + with self.data._lock: + self.data.twd_deg = twd_deg + self.data.wind_time = time.time() + if self.verbose: + print(f"TWD: {twd_deg:.1f}°") + decoded = True + + # Field 5 = Wind Speed (m/s) + if 5 in fields and fields[5].wire_type == WIRE_FIXED32: + val = self.decode_float(fields[5].value) + if val is not None and 0 <= val <= 100: + tws_kts = val * MS_TO_KTS + with self.data._lock: + self.data.tws_kts = tws_kts + if self.verbose: + print(f"TWS: {tws_kts:.1f} kts") + decoded = True + + # Field 6 = Another wind speed (possibly apparent) + if 6 in fields and fields[6].wire_type == WIRE_FIXED32: + val = self.decode_float(fields[6].value) + if val is not None and 0 <= val <= 100: + aws_kts = val * MS_TO_KTS + with self.data._lock: + self.data.aws_kts = aws_kts + decoded = True + + return decoded + + def _extract_depth(self, fields: Dict[int, ProtoField]) -> bool: + """Extract depth from Field 7's children.""" + # Field 1 = Depth in meters + if 1 in fields and fields[1].wire_type == WIRE_FIXED32: + val = self.decode_float(fields[1].value) + # Validate: reasonable depth range (0-1000 meters) + if val is not None and 0 < val <= 1000: + depth_ft = val / FEET_TO_M # Convert meters to feet + with self.data._lock: + self.data.depth_ft = depth_ft + self.data.depth_time = time.time() + if self.verbose: + print(f"Depth: {depth_ft:.1f} ft ({val:.2f} m)") + return True + return False + + def _extract_temperature(self, fields: Dict[int, ProtoField]) -> bool: + """Extract temperature from Field 15's children.""" + decoded = False + + # Field 3 = Air Temperature (Kelvin) + if 3 in fields and fields[3].wire_type == WIRE_FIXED32: + val = self.decode_float(fields[3].value) + # Validate: reasonable temp range (200-350 K = -73 to 77°C) + if val is not None and 200 <= val <= 350: + temp_c = val - KELVIN_OFFSET + with self.data._lock: + self.data.air_temp_c = temp_c + if self.verbose: + print(f"Air Temp: {temp_c:.1f}°C ({val:.1f} K)") + decoded = True + + # Field 9 = Water Temperature (Kelvin) + if 9 in fields and fields[9].wire_type == WIRE_FIXED32: + val = self.decode_float(fields[9].value) + # Validate: reasonable water temp range (270-320 K = -3 to 47°C) + if val is not None and 270 <= val <= 320: + temp_c = val - KELVIN_OFFSET + with self.data._lock: + self.data.water_temp_c = temp_c + if self.verbose: + print(f"Water Temp: {temp_c:.1f}°C ({val:.1f} K)") + decoded = True + + return decoded + + def _extract_tanks(self, tank_fields: List[ProtoField]) -> bool: + """Extract tank levels from Field 16 (repeated).""" + decoded = False + unknown_fuel_idx = 0 # Counter for fuel tanks without IDs + + for tank_field in tank_fields: + if not tank_field.children: + continue + + children = tank_field.children + tank_id = None + level = None + status = None + + # Field 1 = Tank ID (varint) + if 1 in children and children[1].wire_type == WIRE_VARINT: + tank_id = children[1].value + + # Field 2 = Status (varint) + if 2 in children and children[2].wire_type == WIRE_VARINT: + status = children[2].value + + # Field 3 = Tank Level percentage (float) + if 3 in children and children[3].wire_type == WIRE_FIXED32: + val = self.decode_float(children[3].value) + # Validate: 0-100% range + if val is not None and 0 <= val <= 100: + level = val + + # If we have a level but no tank_id, try to infer it + if tank_id is None and level is not None: + if status == TANK_STATUS_WASTE: + # Black/gray water tank - only tank with status=5 + tank_id = 100 + elif status is None: + # Port Fuel is the ONLY tank with neither ID nor status + # (Stbd Fuel has ID=1, water tanks have status=1) + tank_id = 2 # Port Fuel + + if tank_id is not None and level is not None: + with self.data._lock: + self.data.tanks[tank_id] = level + self.data.tank_time = time.time() + if self.verbose: + print(f"Tank {tank_id}: {level:.1f}%") + decoded = True + + return decoded + + def _extract_batteries(self, battery_fields: List[ProtoField]) -> bool: + """Extract battery voltages from Field 20 (repeated).""" + decoded = False + + for battery_field in battery_fields: + if not battery_field.children: + continue + + children = battery_field.children + battery_id = None + voltage = None + + # Field 1 = Battery ID (varint) + if 1 in children and children[1].wire_type == WIRE_VARINT: + battery_id = children[1].value + + # Field 3 = Voltage (float) + if 3 in children and children[3].wire_type == WIRE_FIXED32: + val = self.decode_float(children[3].value) + # Validate: reasonable voltage range (10-60V covers 12V, 24V, 48V systems) + if val is not None and 10 <= val <= 60: + voltage = val + + if battery_id is not None and voltage is not None: + with self.data._lock: + self.data.batteries[battery_id] = voltage + self.data.battery_time = time.time() + if self.verbose: + print(f"Battery {battery_id}: {voltage:.2f}V") + decoded = True + + return decoded + + def _extract_engine_batteries(self, engine_fields: List[ProtoField]) -> bool: + """Extract engine battery voltages from Field 14 (repeated). + + Engine data structure: + Field 14.1 (varint): Engine ID (0=Port, 1=Starboard) + Field 14.3 (message): Engine sensor data + Field 14.3.4 (float): Battery voltage + """ + decoded = False + + for engine_field in engine_fields: + if not engine_field.children: + continue + + children = engine_field.children + + # Field 1 = Engine ID (varint), default 0 (Port) if not present + engine_id = 0 + if 1 in children and children[1].wire_type == WIRE_VARINT: + engine_id = children[1].value + + # Field 3 = Engine sensor data (nested message) + if 3 in children and children[3].wire_type == WIRE_LENGTH: + sensor_data = children[3].value + # Parse the nested message to get Field 4 (voltage) + sensor_parser = ProtobufParser(sensor_data) + sensor_fields = sensor_parser.parse_message() + + if 4 in sensor_fields and sensor_fields[4].wire_type == WIRE_FIXED32: + val = self.decode_float(sensor_fields[4].value) + # Validate: reasonable voltage range (10-60V) + if val is not None and 10 <= val <= 60: + # Use battery_id = 1000 + engine_id to distinguish from house batteries + battery_id = 1000 + engine_id + with self.data._lock: + self.data.batteries[battery_id] = val + self.data.battery_time = time.time() + if self.verbose: + print(f"Engine {engine_id} Battery: {val:.2f}V") + decoded = True + + return decoded + + +class MulticastListener: + """Listens on Raymarine multicast groups.""" + + def __init__(self, decoder: RaymarineDecoder, interface_ip: str, + groups: List[Tuple[str, int]] = None): + """Initialize listener. + + Args: + decoder: RaymarineDecoder instance + interface_ip: IP address of network interface + groups: List of (group, port) tuples. Defaults to MULTICAST_GROUPS. + """ + self.decoder = decoder + self.interface_ip = interface_ip + self.groups = groups if groups is not None else MULTICAST_GROUPS + self.running = False + self.sockets = [] + self.threads = [] + + def _create_socket(self, group: str, port: int) -> Optional[socket.socket]: + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, 'SO_REUSEPORT'): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind(('', port)) + mreq = struct.pack("4s4s", socket.inet_aton(group), socket.inet_aton(self.interface_ip)) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + sock.settimeout(1.0) + return sock + except Exception as e: + print(f"Error creating socket for {group}:{port}: {e}") + return None + + def _listen(self, sock: socket.socket, group: str, port: int): + group_key = f"{group}:{port}" + while self.running: + try: + data, addr = sock.recvfrom(65535) + self.decoder.decode_packet(data, group=group_key) + except socket.timeout: + continue + except Exception as e: + if self.running: + print(f"Error on {group}:{port}: {e}") + + def start(self): + self.running = True + for group, port in self.groups: + sock = self._create_socket(group, port) + if sock: + self.sockets.append(sock) + t = threading.Thread(target=self._listen, args=(sock, group, port), daemon=True) + t.start() + self.threads.append(t) + print(f"Listening on {group}:{port}") + print(f"Active groups: {len(self.groups)} (threads: {len(self.threads)})") + + def stop(self): + self.running = False + for t in self.threads: + t.join(timeout=2) + for s in self.sockets: + try: + s.close() + except: + pass + + +def read_pcap(filename: str): + """Read packets from pcap file.""" + packets = [] + with open(filename, 'rb') as f: + header = f.read(24) + magic = struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + + +def display_dashboard(data: SensorData): + """Display live dashboard.""" + print("\033[2J\033[H", end="") # Clear screen + + d = data.to_dict() + ts = datetime.now().strftime("%H:%M:%S") + + print("=" * 60) + print(f" RAYMARINE DECODER (Protobuf) {ts}") + print("=" * 60) + + pos = d["position"] + if pos["latitude"] and pos["longitude"]: + print(f" GPS: {pos['latitude']:.6f}, {pos['longitude']:.6f}") + else: + print(" GPS: No data") + + nav = d["navigation"] + if nav["heading_deg"] is not None or nav["cog_deg"] is not None or nav["sog_kts"] is not None: + hdg = f"{nav['heading_deg']}°" if nav['heading_deg'] is not None else "---" + cog = f"{nav['cog_deg']}°" if nav['cog_deg'] is not None else "---" + sog = f"{nav['sog_kts']} kts" if nav['sog_kts'] is not None else "---" + print(f" Heading: {hdg} COG: {cog} SOG: {sog}") + else: + print(" Nav: No data") + + wind = d["wind"] + if wind["true_direction_deg"] or wind["true_speed_kts"]: + twd = f"{wind['true_direction_deg']}°" if wind['true_direction_deg'] else "---" + tws = f"{wind['true_speed_kts']} kts" if wind['true_speed_kts'] else "---" + print(f" Wind: {tws} @ {twd} (true)") + else: + print(" Wind: No data") + + depth = d["depth"] + if depth["feet"]: + print(f" Depth: {depth['feet']} ft ({depth['meters']} m)") + else: + print(" Depth: No data") + + temp = d["temperature"] + if temp["air_c"] is not None or temp["water_c"] is not None: + if temp['air_c'] is not None: + air_f = temp['air_c'] * 9/5 + 32 + air = f"{temp['air_c']}°C / {air_f:.1f}°F" + else: + air = "---" + if temp['water_c'] is not None: + water_f = temp['water_c'] * 9/5 + 32 + water = f"{temp['water_c']}°C / {water_f:.1f}°F" + else: + water = "---" + print(f" Temp: Air {air}, Water {water}") + else: + print(" Temp: No data") + + tanks = d["tanks"] + if tanks: + tank_parts = [] + for tid, tank_info in sorted(tanks.items(), key=lambda x: int(x[0])): + name = tank_info["name"] + lvl = tank_info["level_pct"] + capacity = tank_info["capacity_gal"] + if capacity: + gallons = capacity * lvl / 100 + tank_parts.append(f"{name}: {lvl}% ({gallons:.0f}gal)") + else: + tank_parts.append(f"{name}: {lvl}%") + print(f" Tanks: {', '.join(tank_parts)}") + else: + print(" Tanks: No data") + + batteries = d["batteries"] + if batteries: + battery_parts = [] + for bid, battery_info in sorted(batteries.items(), key=lambda x: int(x[0])): + name = battery_info["name"] + voltage = battery_info["voltage_v"] + battery_parts.append(f"{name}: {voltage}V") + print(f" Batts: {', '.join(battery_parts)}") + else: + print(" Batts: No data") + + print("-" * 60) + stats = d["stats"] + print(f" Packets: {stats['packets']} Decoded: {stats['decoded']} Uptime: {stats['uptime_s']}s") + + # Show per-group statistics + with data._lock: + if data.group_stats: + print("-" * 60) + print(" GROUP STATISTICS:") + for group_key, gstats in sorted(data.group_stats.items()): + fields_str = ", ".join(sorted(gstats["fields"])) if gstats["fields"] else "none" + pct = (gstats["decoded"] / gstats["packets"] * 100) if gstats["packets"] > 0 else 0 + print(f" {group_key}: {gstats['packets']} pkts, {gstats['decoded']} decoded ({pct:.0f}%)") + print(f" Fields: {fields_str}") + print("=" * 60) + + +def main(): + parser = argparse.ArgumentParser( + description="Raymarine Protobuf Decoder", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Group presets: + primary - Only 226.192.206.102:2565 (all sensor data from Data Master) + minimal - Primary + 239.2.1.1:2154 (backup for tank/engine data) + all - All 7 multicast groups (original behavior, high CPU) + +Examples: + %(prog)s -i 198.18.5.5 --groups primary # Test with single group + %(prog)s -i 198.18.5.5 --groups minimal # Test with 2 groups + %(prog)s -i 198.18.5.5 --groups all # Use all groups (default) +""" + ) + parser.add_argument('-i', '--interface', help='Interface IP for multicast') + parser.add_argument('--pcap', help='Read from pcap file') + parser.add_argument('--json', action='store_true', help='JSON output') + parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output') + parser.add_argument('-g', '--groups', choices=['primary', 'minimal', 'all'], + default='all', help='Multicast group preset (default: all)') + args = parser.parse_args() + + if not args.pcap and not args.interface: + parser.error("Either --interface or --pcap is required") + + # Get selected multicast groups + selected_groups = GROUP_PRESETS[args.groups] + print(f"Using group preset: {args.groups} ({len(selected_groups)} groups)") + + sensor_data = SensorData() + decoder = RaymarineDecoder(sensor_data, verbose=args.verbose) + + if args.pcap: + print(f"Reading {args.pcap}...") + packets = read_pcap(args.pcap) + print(f"Processing {len(packets)} packets...\n") + + for pkt in packets: + decoder.decode_packet(pkt) + + print("Final state:") + print(json.dumps(sensor_data.to_dict(), indent=2)) + + else: + listener = MulticastListener(decoder, args.interface, groups=selected_groups) + try: + listener.start() + while True: + if args.json: + print(json.dumps(sensor_data.to_dict())) + else: + display_dashboard(sensor_data) + time.sleep(0.5) + except KeyboardInterrupt: + print("\nStopping...") + finally: + listener.stop() + print("\nFinal state:") + print(json.dumps(sensor_data.to_dict(), indent=2)) + + # Print final group statistics summary + print("\n" + "=" * 60) + print("GROUP STATISTICS SUMMARY") + print("=" * 60) + with sensor_data._lock: + for group_key, gstats in sorted(sensor_data.group_stats.items()): + fields_str = ", ".join(sorted(gstats["fields"])) if gstats["fields"] else "none" + print(f"\n{group_key}:") + print(f" Packets received: {gstats['packets']}") + print(f" Packets decoded: {gstats['decoded']}") + print(f" Data types found: {fields_str}") + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/debug/protobuf_parser.py b/axiom-nmea/debug/protobuf_parser.py new file mode 100644 index 0000000..9e3c798 --- /dev/null +++ b/axiom-nmea/debug/protobuf_parser.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +""" +Proper protobuf wire format parser for Raymarine packets. +Decodes the nested message structure to understand the protocol. +""" + +import struct +from dataclasses import dataclass +from typing import List, Tuple, Optional, Any + +# Wire types +WIRE_VARINT = 0 # int32, int64, uint32, uint64, sint32, sint64, bool, enum +WIRE_FIXED64 = 1 # fixed64, sfixed64, double +WIRE_LENGTH = 2 # string, bytes, embedded messages, packed repeated fields +WIRE_FIXED32 = 5 # fixed32, sfixed32, float + +WIRE_NAMES = { + 0: 'varint', + 1: 'fixed64', + 2: 'length-delim', + 5: 'fixed32', +} + +@dataclass +class ProtoField: + """Represents a decoded protobuf field.""" + field_num: int + wire_type: int + offset: int + length: int + raw_value: bytes + decoded_value: Any = None + children: List['ProtoField'] = None + + def __post_init__(self): + if self.children is None: + self.children = [] + + +class ProtobufDecoder: + """Decodes protobuf wire format without a schema.""" + + def __init__(self, data: bytes): + self.data = data + self.pos = 0 + + def decode_varint(self) -> Tuple[int, int]: + """Decode a varint, return (value, bytes_consumed).""" + result = 0 + shift = 0 + start = self.pos + + while self.pos < len(self.data): + byte = self.data[self.pos] + self.pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + if shift > 63: + break + + return result, self.pos - start + + def decode_fixed64(self) -> bytes: + """Decode 8 bytes (fixed64/double).""" + value = self.data[self.pos:self.pos + 8] + self.pos += 8 + return value + + def decode_fixed32(self) -> bytes: + """Decode 4 bytes (fixed32/float).""" + value = self.data[self.pos:self.pos + 4] + self.pos += 4 + return value + + def decode_length_delimited(self) -> bytes: + """Decode length-delimited field (string, bytes, nested message).""" + length, _ = self.decode_varint() + value = self.data[self.pos:self.pos + length] + self.pos += length + return value + + def decode_field(self) -> Optional[ProtoField]: + """Decode a single protobuf field.""" + if self.pos >= len(self.data): + return None + + start_offset = self.pos + + # Decode tag + tag, _ = self.decode_varint() + field_num = tag >> 3 + wire_type = tag & 0x07 + + # Sanity check + if field_num == 0 or field_num > 536870911: # Max field number + return None + + try: + if wire_type == WIRE_VARINT: + value, _ = self.decode_varint() + raw = self.data[start_offset:self.pos] + return ProtoField(field_num, wire_type, start_offset, + self.pos - start_offset, raw, value) + + elif wire_type == WIRE_FIXED64: + raw = self.decode_fixed64() + # Try to decode as double + try: + double_val = struct.unpack(' List[ProtoField]: + """Decode all fields in the buffer.""" + fields = [] + while self.pos < len(self.data): + field = self.decode_field() + if field is None: + break + fields.append(field) + return fields + + +def try_decode_nested(field: ProtoField, depth: int = 0, max_depth: int = 5) -> bool: + """Try to decode a length-delimited field as a nested message.""" + if field.wire_type != WIRE_LENGTH or depth >= max_depth: + return False + + if len(field.raw_value) < 2: + return False + + # Try to decode as nested protobuf + decoder = ProtobufDecoder(field.raw_value) + children = [] + + try: + while decoder.pos < len(decoder.data): + child = decoder.decode_field() + if child is None: + break + # Recursively try to decode nested messages + if child.wire_type == WIRE_LENGTH: + try_decode_nested(child, depth + 1, max_depth) + children.append(child) + + # Only consider it a valid nested message if we decoded most of the data + if children and decoder.pos >= len(decoder.data) * 0.8: + field.children = children + return True + except: + pass + + return False + + +def print_field(field: ProtoField, indent: int = 0, show_raw: bool = False): + """Pretty print a protobuf field.""" + prefix = " " * indent + wire_name = WIRE_NAMES.get(field.wire_type, f'unknown({field.wire_type})') + + # Format value based on type + if field.wire_type == WIRE_VARINT: + value_str = f"{field.decoded_value}" + elif field.wire_type == WIRE_FIXED64: + if isinstance(field.decoded_value, float): + value_str = f"{field.decoded_value:.6f}" + # Check if it could be coordinates + if -90 <= field.decoded_value <= 90: + value_str += " (could be lat)" + elif -180 <= field.decoded_value <= 180: + value_str += " (could be lon)" + else: + value_str = field.raw_value.hex() + elif field.wire_type == WIRE_FIXED32: + if isinstance(field.decoded_value, float): + value_str = f"{field.decoded_value:.4f}" + # Check if it could be angle in radians + if 0 <= field.decoded_value <= 6.5: + deg = field.decoded_value * 57.2958 + value_str += f" ({deg:.1f}°)" + else: + value_str = field.raw_value.hex() + elif field.wire_type == WIRE_LENGTH: + if field.children: + value_str = f"[nested message, {len(field.children)} fields]" + else: + # Try to show as string if printable + try: + text = field.raw_value.decode('ascii') + if all(32 <= ord(c) < 127 or c in '\n\r\t' for c in text): + value_str = f'"{text}"' + else: + value_str = f"[{len(field.raw_value)} bytes]" + except: + value_str = f"[{len(field.raw_value)} bytes]" + else: + value_str = field.raw_value.hex() + + print(f"{prefix}field {field.field_num:2d} ({wire_name:12s}) @ 0x{field.offset:04x}: {value_str}") + + if show_raw and field.wire_type == WIRE_LENGTH and not field.children: + # Show hex dump for non-nested length-delimited fields + hex_str = ' '.join(f'{b:02x}' for b in field.raw_value[:32]) + if len(field.raw_value) > 32: + hex_str += ' ...' + print(f"{prefix} raw: {hex_str}") + + # Print children + for child in field.children: + print_field(child, indent + 1, show_raw) + + +def analyze_packet(data: bytes, show_raw: bool = False): + """Analyze a single packet.""" + print(f"\nPacket length: {len(data)} bytes") + + # Check for fixed header + if len(data) > 20: + header = data[:20] + print(f"Header (first 20 bytes): {header.hex()}") + + # Look for protobuf start (usually 0x0a = field 1, length-delimited) + proto_start = None + for i in range(0, min(24, len(data))): + if data[i] == 0x0a: # field 1, wire type 2 + proto_start = i + break + + if proto_start is not None: + print(f"Protobuf likely starts at offset 0x{proto_start:04x}") + proto_data = data[proto_start:] + else: + print("No clear protobuf start found, trying from offset 0") + proto_data = data + else: + proto_data = data + + # Decode protobuf + decoder = ProtobufDecoder(proto_data) + fields = decoder.decode_all() + + print(f"\nDecoded {len(fields)} top-level fields:") + print("-" * 60) + + for field in fields: + # Try to decode nested messages + if field.wire_type == WIRE_LENGTH: + try_decode_nested(field) + print_field(field, show_raw=show_raw) + + +def read_pcap(filename): + """Read packets from pcap file.""" + packets = [] + with open(filename, 'rb') as f: + header = f.read(24) + magic = struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + + +if __name__ == "__main__": + import sys + + pcap_file = sys.argv[1] if len(sys.argv) > 1 else "raymarine_sample_TWD_62-70_HDG_29-35.pcap" + + print(f"Reading {pcap_file}...") + packets = read_pcap(pcap_file) + print(f"Loaded {len(packets)} packets") + + # Group by size + by_size = {} + for pkt in packets: + pkt_len = len(pkt) + if pkt_len not in by_size: + by_size[pkt_len] = [] + by_size[pkt_len].append(pkt) + + # Analyze one packet of each key size + target_sizes = [344, 446, 788, 888, 1472] + + for size in target_sizes: + if size in by_size: + print("\n" + "=" * 70) + print(f"ANALYZING {size}-BYTE PACKET") + print("=" * 70) + analyze_packet(by_size[size][0], show_raw=True) diff --git a/axiom-nmea/debug/raymarine_decoder.py b/axiom-nmea/debug/raymarine_decoder.py new file mode 100755 index 0000000..d2a4fb9 --- /dev/null +++ b/axiom-nmea/debug/raymarine_decoder.py @@ -0,0 +1,806 @@ +#!/usr/bin/env python3 +""" +Raymarine LightHouse Network Decoder + +Decodes sensor data from Raymarine AXIOM MFDs broadcast over UDP multicast. +The protocol uses Protocol Buffers binary encoding (not standard NMEA 0183). + +Usage: + python raymarine_decoder.py -i 198.18.5.5 + python raymarine_decoder.py -i 198.18.5.5 --json + python raymarine_decoder.py --pcap raymarine_sample.pcap + +Multicast Groups: + 226.192.206.98:2561 - Navigation sensors + 226.192.206.99:2562 - Heartbeat/status + 226.192.206.102:2565 - Mixed sensor data + 226.192.219.0:3221 - Display sync + +Author: Reverse-engineered from Raymarine network captures +""" + +import argparse +import json +import os +import socket +import struct +import sys +import threading +import time +from collections import deque +from datetime import datetime +from typing import Dict, List, Optional, Tuple, Any + + +# Raymarine multicast configuration +MULTICAST_GROUPS = [ + ("226.192.206.98", 2561), # Navigation sensors + ("226.192.206.99", 2562), # Heartbeat/status + ("226.192.206.102", 2565), # Mixed sensor data (primary) + ("226.192.219.0", 3221), # Display sync +] + +# Conversion constants +RADIANS_TO_DEGREES = 57.2957795131 +MS_TO_KNOTS = 1.94384449 +FEET_TO_METERS = 0.3048 + + +class SensorData: + """Holds the current state of all decoded sensor values.""" + + def __init__(self): + self.latitude: Optional[float] = None + self.longitude: Optional[float] = None + self.heading_deg: Optional[float] = None + self.wind_speed_kts: Optional[float] = None + self.wind_direction_deg: Optional[float] = None + self.depth_ft: Optional[float] = None + self.water_temp_c: Optional[float] = None + self.air_temp_c: Optional[float] = None + self.sog_kts: Optional[float] = None # Speed over ground + self.cog_deg: Optional[float] = None # Course over ground + + # Timestamps for freshness tracking + self.gps_time: float = 0 + self.heading_time: float = 0 + self.wind_time: float = 0 + self.depth_time: float = 0 + self.temp_time: float = 0 + + # Statistics + self.packet_count: int = 0 + self.gps_count: int = 0 + self.start_time: float = time.time() + + # Thread safety + self.lock = threading.Lock() + + def to_dict(self) -> Dict[str, Any]: + """Convert sensor data to dictionary for JSON output.""" + with self.lock: + return { + "timestamp": datetime.now().isoformat(), + "position": { + "latitude": self.latitude, + "longitude": self.longitude, + "age_seconds": time.time() - self.gps_time if self.gps_time else None, + }, + "navigation": { + "heading_deg": self.heading_deg, + "sog_kts": self.sog_kts, + "cog_deg": self.cog_deg, + }, + "wind": { + "speed_kts": self.wind_speed_kts, + "direction_deg": self.wind_direction_deg, + }, + "depth": { + "feet": self.depth_ft, + "meters": self.depth_ft * FEET_TO_METERS if self.depth_ft else None, + }, + "temperature": { + "water_c": self.water_temp_c, + "air_c": self.air_temp_c, + }, + "statistics": { + "packets_received": self.packet_count, + "gps_packets": self.gps_count, + "uptime_seconds": time.time() - self.start_time, + } + } + + +class ProtobufDecoder: + """ + Decodes Raymarine's protobuf-like binary format. + + Wire types: + 0 = Varint + 1 = 64-bit (fixed64, double) + 2 = Length-delimited (string, bytes, nested message) + 5 = 32-bit (fixed32, float) + """ + + @staticmethod + def decode_varint(data: bytes, offset: int) -> Tuple[int, int]: + """Decode a protobuf varint, return (value, bytes_consumed).""" + result = 0 + shift = 0 + consumed = 0 + + while offset + consumed < len(data): + byte = data[offset + consumed] + result |= (byte & 0x7F) << shift + consumed += 1 + if not (byte & 0x80): + break + shift += 7 + if shift > 63: + break + + return result, consumed + + @staticmethod + def decode_double(data: bytes, offset: int) -> Optional[float]: + """Decode a little-endian 64-bit double.""" + if offset + 8 > len(data): + return None + try: + return struct.unpack(' Optional[float]: + """Decode a little-endian 32-bit float.""" + if offset + 4 > len(data): + return None + try: + return struct.unpack(' bool: + """Check if value is a valid latitude.""" + return -90 <= val <= 90 + + @staticmethod + def is_valid_longitude(val: float) -> bool: + """Check if value is a valid longitude.""" + return -180 <= val <= 180 + + @staticmethod + def is_valid_angle_radians(val: float) -> bool: + """Check if value is a valid angle in radians (0 to 2*pi).""" + return 0 <= val <= 6.5 # Slightly more than 2*pi for tolerance + + @staticmethod + def is_valid_speed_ms(val: float) -> bool: + """Check if value is a reasonable speed in m/s (0 to ~50 m/s = ~100 kts).""" + return 0 <= val <= 60 + + +class RaymarineDecoder: + """ + Main decoder for Raymarine network packets. + + Uses GPS-anchored parsing strategy: + 1. Find GPS using reliable 0x09/0x11 pattern at offset ~0x0032 + 2. Extract other values at known offsets relative to GPS or packet start + """ + + def __init__(self, sensor_data: SensorData, verbose: bool = False): + self.sensor_data = sensor_data + self.verbose = verbose + self.pb = ProtobufDecoder() + + # Packet size categories for different parsing strategies + self.SMALL_PACKETS = range(0, 200) + self.MEDIUM_PACKETS = range(200, 600) + self.LARGE_PACKETS = range(600, 1200) + self.XLARGE_PACKETS = range(1200, 3000) + + def decode_packet(self, data: bytes, source: Tuple[str, int]) -> bool: + """ + Decode a single UDP packet. + Returns True if any useful data was extracted. + """ + with self.sensor_data.lock: + self.sensor_data.packet_count += 1 + + if len(data) < 50: + return False # Too small to contain useful data + + decoded_something = False + + # Try GPS extraction (most reliable) + if self._extract_gps(data): + decoded_something = True + + # Try extracting other sensor data based on packet size + pkt_len = len(data) + + if pkt_len in self.LARGE_PACKETS or pkt_len in self.XLARGE_PACKETS: + # Large packets typically have full sensor data + if self._extract_navigation(data): + decoded_something = True + if self._extract_wind(data): + decoded_something = True + if self._extract_depth(data): + decoded_something = True + if self._extract_temperature(data): + decoded_something = True + + elif pkt_len in self.MEDIUM_PACKETS: + # Medium packets may have partial data + if self._extract_wind(data): + decoded_something = True + if self._extract_depth(data): + decoded_something = True + + return decoded_something + + def _extract_gps(self, data: bytes) -> bool: + """ + Extract GPS coordinates using the 0x09/0x11 pattern. + + Pattern: + 0x09 [8-byte latitude double] 0x11 [8-byte longitude double] + + Returns True if valid GPS was found. + """ + # Scan for the GPS pattern starting around offset 0x30 + search_start = 0x20 + search_end = min(len(data) - 18, 0x100) + + for offset in range(search_start, search_end): + if data[offset] != 0x09: + continue + + # Check if 0x11 follows at expected position + lon_tag_offset = offset + 9 + if lon_tag_offset >= len(data) or data[lon_tag_offset] != 0x11: + continue + + # Decode latitude and longitude + lat = self.pb.decode_double(data, offset + 1) + lon = self.pb.decode_double(data, lon_tag_offset + 1) + + if lat is None or lon is None: + continue + + # Validate coordinates + if not self.pb.is_valid_latitude(lat) or not self.pb.is_valid_longitude(lon): + continue + + # Additional sanity check: filter out obviously wrong values + # Most readings should be reasonable coordinates, not near 0,0 + if abs(lat) < 0.1 and abs(lon) < 0.1: + continue + + with self.sensor_data.lock: + self.sensor_data.latitude = lat + self.sensor_data.longitude = lon + self.sensor_data.gps_time = time.time() + self.sensor_data.gps_count += 1 + + if self.verbose: + print(f"GPS: {lat:.6f}, {lon:.6f} (offset 0x{offset:04x})") + + return True + + return False + + def _extract_navigation(self, data: bytes) -> bool: + """ + Extract heading, SOG, COG from packet. + These are typically 32-bit floats in radians. + """ + found = False + + # Look for heading at known offsets for large packets + heading_offsets = [0x006f, 0x00d4, 0x0073, 0x00d8] + + for offset in heading_offsets: + if offset + 4 > len(data): + continue + + # Check for float tag (wire type 5) + if offset > 0 and (data[offset - 1] & 0x07) == 5: + val = self.pb.decode_float(data, offset) + if val and self.pb.is_valid_angle_radians(val): + heading_deg = val * RADIANS_TO_DEGREES + with self.sensor_data.lock: + self.sensor_data.heading_deg = heading_deg % 360 + self.sensor_data.heading_time = time.time() + found = True + break + + return found + + def _extract_wind(self, data: bytes) -> bool: + """ + Extract wind speed and direction. + Wind speed is in m/s, direction in radians. + + Known offsets by packet size (discovered via pcap analysis): + - 344 bytes: speed @ 0x00a5, dir @ 0x00a0 + - 446 bytes: speed @ 0x00ac, dir @ 0x00a7 + - 788 bytes: speed @ 0x00ca, dir @ 0x00c5 + - 888 bytes: speed @ 0x00ca, dir @ 0x00c5 + - 931 bytes: speed @ 0x00ca, dir @ 0x00c5 + - 1031 bytes: speed @ 0x00ca, dir @ 0x00c5 + - 1472 bytes: speed @ 0x0101, dir @ 0x00fc + + Note: 878-byte packets do NOT contain wind data at these offsets. + """ + pkt_len = len(data) + + # Define offset pairs (speed_offset, dir_offset) for SPECIFIC packet sizes + # Only process packet sizes known to contain wind data + offset_pairs = None + + if pkt_len == 344: + offset_pairs = [(0x00a5, 0x00a0)] + elif pkt_len == 446: + offset_pairs = [(0x00ac, 0x00a7)] + elif pkt_len in (788, 888, 931, 1031): + offset_pairs = [(0x00ca, 0x00c5)] + elif pkt_len == 1472: + offset_pairs = [(0x0101, 0x00fc)] + + # Skip unknown packet sizes to avoid garbage values + if offset_pairs is None: + return False + + for speed_offset, dir_offset in offset_pairs: + if speed_offset + 4 > pkt_len or dir_offset + 4 > pkt_len: + continue + + speed_val = self.pb.decode_float(data, speed_offset) + dir_val = self.pb.decode_float(data, dir_offset) + + if speed_val is None or dir_val is None: + continue + + # Validate: speed 0.1-50 m/s (~0.2-97 kts), direction 0-2*pi radians + if not (0.1 < speed_val < 50): + continue + if not (0 <= dir_val <= 6.5): + continue + + # Convert and store + with self.sensor_data.lock: + self.sensor_data.wind_speed_kts = speed_val * MS_TO_KNOTS + self.sensor_data.wind_direction_deg = (dir_val * RADIANS_TO_DEGREES) % 360 + self.sensor_data.wind_time = time.time() + return True + + return False + + def _extract_depth(self, data: bytes) -> bool: + """ + Extract depth value (in feet, stored as 64-bit double). + Depth is tagged with field 5 (0x29) or field 11 (0x59) wire type 1. + """ + # Search for depth by looking for wire type 1 tags with field 5 or 11 + # Tag format: (field_number << 3) | wire_type + # Field 5, wire type 1 = (5 << 3) | 1 = 0x29 + # Field 11, wire type 1 = (11 << 3) | 1 = 0x59 + + depth_tags = [0x29, 0x59] # Field 5 and 11, wire type 1 + + for offset in range(0x40, min(len(data) - 9, 0x300)): + tag = data[offset] + if tag not in depth_tags: + continue + + val = self.pb.decode_double(data, offset + 1) + if val is None: + continue + + # Reasonable depth range: 0.5 to 500 feet + if 0.5 < val < 500: + with self.sensor_data.lock: + self.sensor_data.depth_ft = val + self.sensor_data.depth_time = time.time() + return True + + # Fallback: scan for any reasonable depth-like double values + # in larger packets where we have more sensor data + if len(data) > 800: + for offset in range(0x80, min(len(data) - 9, 0x200)): + # Only check positions that look like protobuf fields + tag = data[offset] + if (tag & 0x07) != 1: # Wire type 1 (double) + continue + + val = self.pb.decode_double(data, offset + 1) + if val is None: + continue + + # Typical depth range for Florida Keys: 3-50 feet + if 2 < val < 100: + with self.sensor_data.lock: + self.sensor_data.depth_ft = val + self.sensor_data.depth_time = time.time() + return True + + return False + + def _extract_temperature(self, data: bytes) -> bool: + """ + Extract temperature values (water and air). + Temperature encoding is not yet fully understood. + Might be in Kelvin, Celsius, or Fahrenheit. + + Note: Temperature extraction is experimental and may not produce + reliable results without the proprietary protobuf schema. + """ + # Temperature extraction is currently unreliable + # The protocol documentation notes temperature has not been found + # Returning False to avoid displaying garbage values + # TODO: Implement when temperature field offsets are discovered + + # Search for temperature-like values with stricter validation + # Only look at specific wire type 1 (double) fields + for offset in range(0x50, min(len(data) - 9, 0x200)): + # Must be preceded by a wire type 1 tag + tag = data[offset] + if (tag & 0x07) != 1: # Wire type 1 = 64-bit + continue + + field_num = tag >> 3 + # Temperature fields are likely in a reasonable field number range + if field_num < 1 or field_num > 30: + continue + + val = self.pb.decode_double(data, offset + 1) + if val is None: + continue + + # Very strict validation for Kelvin range (water temp 15-35°C) + if 288 < val < 308: # 15°C to 35°C in Kelvin + temp_c = val - 273.15 + with self.sensor_data.lock: + if self.sensor_data.water_temp_c is None: + self.sensor_data.water_temp_c = temp_c + self.sensor_data.temp_time = time.time() + return True + + return False + + +class MulticastListener: + """ + Listens on multiple multicast groups and feeds packets to the decoder. + """ + + def __init__(self, decoder: RaymarineDecoder, interface_ip: str, + groups: List[Tuple[str, int]] = None): + self.decoder = decoder + self.interface_ip = interface_ip + self.groups = groups or MULTICAST_GROUPS + self.sockets: List[socket.socket] = [] + self.running = False + self.threads: List[threading.Thread] = [] + + def _create_socket(self, group: str, port: int) -> Optional[socket.socket]: + """Create and configure a multicast socket.""" + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + # Try SO_REUSEPORT if available (Linux) + if hasattr(socket, 'SO_REUSEPORT'): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + + # Bind to the port + sock.bind(('', port)) + + # Join multicast group + mreq = struct.pack("4s4s", + socket.inet_aton(group), + socket.inet_aton(self.interface_ip)) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + + # Set receive timeout + sock.settimeout(1.0) + + return sock + except Exception as e: + print(f"Error creating socket for {group}:{port}: {e}", file=sys.stderr) + return None + + def _listener_thread(self, sock: socket.socket, group: str, port: int): + """Thread function to listen on a single multicast group.""" + while self.running: + try: + data, addr = sock.recvfrom(65535) + self.decoder.decode_packet(data, addr) + except socket.timeout: + continue + except Exception as e: + if self.running: + print(f"Error receiving on {group}:{port}: {e}", file=sys.stderr) + + def start(self): + """Start listening on all multicast groups.""" + self.running = True + + for group, port in self.groups: + sock = self._create_socket(group, port) + if sock: + self.sockets.append(sock) + thread = threading.Thread( + target=self._listener_thread, + args=(sock, group, port), + daemon=True + ) + thread.start() + self.threads.append(thread) + print(f"Listening on {group}:{port}") + + if not self.sockets: + raise RuntimeError("Failed to create any multicast sockets") + + def stop(self): + """Stop listening and clean up.""" + self.running = False + + for thread in self.threads: + thread.join(timeout=2.0) + + for sock in self.sockets: + try: + sock.close() + except Exception: + pass + + self.sockets = [] + self.threads = [] + + +class PcapReader: + """ + Read packets from a pcap file for offline analysis. + Supports pcap format (not pcapng). + """ + + PCAP_MAGIC = 0xa1b2c3d4 + PCAP_MAGIC_SWAPPED = 0xd4c3b2a1 + + def __init__(self, filename: str): + self.filename = filename + self.swapped = False + + def read_packets(self): + """Generator that yields (timestamp, data) tuples.""" + with open(self.filename, 'rb') as f: + # Read global header + header = f.read(24) + if len(header) < 24: + raise ValueError("Invalid pcap file: too short") + + magic = struct.unpack(' 42: + # Check for IPv4 + if pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + udp_start = 14 + ip_header_len + payload_start = udp_start + 8 + + if payload_start < len(pkt_data): + # Extract source IP + src_ip = '.'.join(str(b) for b in pkt_data[26:30]) + src_port = struct.unpack('!H', pkt_data[udp_start:udp_start+2])[0] + + payload = pkt_data[payload_start:] + yield (ts_sec + ts_usec / 1e6, payload, (src_ip, src_port)) + + +def format_lat_lon(lat: float, lon: float) -> str: + """Format coordinates as degrees and decimal minutes.""" + lat_dir = 'N' if lat >= 0 else 'S' + lon_dir = 'E' if lon >= 0 else 'W' + + lat = abs(lat) + lon = abs(lon) + + lat_deg = int(lat) + lat_min = (lat - lat_deg) * 60 + + lon_deg = int(lon) + lon_min = (lon - lon_deg) * 60 + + return f"{lat_deg:3d}° {lat_min:06.3f}' {lat_dir}, {lon_deg:3d}° {lon_min:06.3f}' {lon_dir}" + + +def display_dashboard(sensor_data: SensorData): + """Display a simple text dashboard.""" + now = time.time() + + # Clear screen + print("\033[2J\033[H", end="") + + # Header + timestamp = datetime.now().strftime("%H:%M:%S") + print("=" * 70) + print(f" RAYMARINE DECODER {timestamp}") + print("=" * 70) + + with sensor_data.lock: + # GPS + if sensor_data.latitude is not None and sensor_data.longitude is not None: + age = now - sensor_data.gps_time + fresh = "OK" if age < 5 else "STALE" + pos_str = format_lat_lon(sensor_data.latitude, sensor_data.longitude) + print(f" GPS: {pos_str} [{fresh}]") + else: + print(" GPS: No data") + + # Heading + if sensor_data.heading_deg is not None: + age = now - sensor_data.heading_time + fresh = "OK" if age < 5 else "STALE" + print(f" Heading: {sensor_data.heading_deg:6.1f}° [{fresh}]") + else: + print(" Heading: No data") + + # Wind + if sensor_data.wind_speed_kts is not None: + age = now - sensor_data.wind_time + fresh = "OK" if age < 5 else "STALE" + dir_str = f"@ {sensor_data.wind_direction_deg:.0f}°" if sensor_data.wind_direction_deg else "" + print(f" Wind: {sensor_data.wind_speed_kts:6.1f} kts {dir_str} [{fresh}]") + else: + print(" Wind: No data") + + # Depth + if sensor_data.depth_ft is not None: + age = now - sensor_data.depth_time + fresh = "OK" if age < 5 else "STALE" + depth_m = sensor_data.depth_ft * FEET_TO_METERS + print(f" Depth: {sensor_data.depth_ft:6.1f} ft ({depth_m:.1f} m) [{fresh}]") + else: + print(" Depth: No data") + + # Temperature + if sensor_data.water_temp_c is not None or sensor_data.air_temp_c is not None: + water = f"{sensor_data.water_temp_c:.1f}°C" if sensor_data.water_temp_c else "---" + air = f"{sensor_data.air_temp_c:.1f}°C" if sensor_data.air_temp_c else "---" + print(f" Temp: Water: {water} Air: {air}") + else: + print(" Temp: No data") + + print("-" * 70) + uptime = now - sensor_data.start_time + print(f" Packets: {sensor_data.packet_count} GPS fixes: {sensor_data.gps_count} Uptime: {uptime:.0f}s") + + print("=" * 70) + print(" Press Ctrl+C to exit") + + +def output_json(sensor_data: SensorData): + """Output sensor data as JSON.""" + print(json.dumps(sensor_data.to_dict(), indent=2)) + + +def main(): + parser = argparse.ArgumentParser( + description="Decode Raymarine LightHouse network data", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + %(prog)s -i 198.18.5.5 Live capture with dashboard + %(prog)s -i 198.18.5.5 --json Live capture with JSON output + %(prog)s --pcap capture.pcap Analyze pcap file + +Multicast Groups: + 226.192.206.98:2561 Navigation sensors + 226.192.206.99:2562 Heartbeat/status + 226.192.206.102:2565 Mixed sensor data + 226.192.219.0:3221 Display sync +""" + ) + + parser.add_argument('-i', '--interface', + help='Interface IP address for multicast binding') + parser.add_argument('--pcap', + help='Read from pcap file instead of live capture') + parser.add_argument('--json', action='store_true', + help='Output as JSON instead of dashboard') + parser.add_argument('--json-interval', type=float, default=1.0, + help='JSON output interval in seconds (default: 1.0)') + parser.add_argument('-v', '--verbose', action='store_true', + help='Verbose output') + parser.add_argument('--group', action='append', nargs=2, + metavar=('IP', 'PORT'), + help='Additional multicast group to listen on') + + args = parser.parse_args() + + # Validate arguments + if not args.pcap and not args.interface: + parser.error("Either --interface or --pcap is required") + + # Initialize sensor data and decoder + sensor_data = SensorData() + decoder = RaymarineDecoder(sensor_data, verbose=args.verbose) + + # Add custom groups if specified + groups = list(MULTICAST_GROUPS) + if args.group: + for ip, port in args.group: + groups.append((ip, int(port))) + + if args.pcap: + # Pcap file analysis + print(f"Reading from {args.pcap}...") + reader = PcapReader(args.pcap) + + packet_count = 0 + for ts, data, source in reader.read_packets(): + decoder.decode_packet(data, source) + packet_count += 1 + + print(f"\nProcessed {packet_count} packets") + print("\nFinal sensor state:") + print(json.dumps(sensor_data.to_dict(), indent=2)) + + else: + # Live capture + listener = MulticastListener(decoder, args.interface, groups) + + try: + listener.start() + print(f"\nListening on interface {args.interface}") + print("Waiting for data...\n") + + while True: + if args.json: + output_json(sensor_data) + else: + display_dashboard(sensor_data) + + time.sleep(args.json_interval if args.json else 0.5) + + except KeyboardInterrupt: + print("\n\nStopping...") + finally: + listener.stop() + + if not args.json: + print("\nFinal sensor state:") + print(json.dumps(sensor_data.to_dict(), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/debug/tank_debug.py b/axiom-nmea/debug/tank_debug.py new file mode 100755 index 0000000..3eba3e3 --- /dev/null +++ b/axiom-nmea/debug/tank_debug.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +""" +Tank Debug - Dump raw Field 16 entries to find missing IDs. +""" + +import struct +import socket +import time +import threading + +WIRE_VARINT = 0 +WIRE_FIXED64 = 1 +WIRE_LENGTH = 2 +WIRE_FIXED32 = 5 + +HEADER_SIZE = 20 + +MULTICAST_GROUPS = [ + ("226.192.206.102", 2565), # Main sensor data with tanks +] + + +class ProtobufParser: + def __init__(self, data: bytes): + self.data = data + self.pos = 0 + + def remaining(self): + return len(self.data) - self.pos + + def read_varint(self) -> int: + result = 0 + shift = 0 + while self.pos < len(self.data): + byte = self.data[self.pos] + self.pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + return result + + def read_fixed32(self) -> bytes: + val = self.data[self.pos:self.pos + 4] + self.pos += 4 + return val + + def read_fixed64(self) -> bytes: + val = self.data[self.pos:self.pos + 8] + self.pos += 8 + return val + + def read_length_delimited(self) -> bytes: + length = self.read_varint() + val = self.data[self.pos:self.pos + length] + self.pos += length + return val + + def parse_all_field16(self): + """Parse and collect ALL Field 16 entries with full detail.""" + entries = [] + + while self.pos < len(self.data): + if self.remaining() < 1: + break + try: + start_pos = self.pos + tag = self.read_varint() + field_num = tag >> 3 + wire_type = tag & 0x07 + + if field_num == 0 or field_num > 1000: + break + + if wire_type == WIRE_VARINT: + value = self.read_varint() + elif wire_type == WIRE_FIXED64: + value = self.read_fixed64() + elif wire_type == WIRE_LENGTH: + value = self.read_length_delimited() + elif wire_type == WIRE_FIXED32: + value = self.read_fixed32() + else: + break + + # If this is Field 16, parse its contents in detail + if field_num == 16 and wire_type == WIRE_LENGTH: + entry = self.parse_tank_entry(value) + entry['raw_hex'] = value.hex() + entry['raw_len'] = len(value) + entries.append(entry) + + except: + break + + return entries + + def parse_tank_entry(self, data: bytes) -> dict: + """Parse a single tank entry and return all fields.""" + entry = {'fields': {}} + pos = 0 + + while pos < len(data): + if pos >= len(data): + break + try: + # Read tag + tag_byte = data[pos] + pos += 1 + + # Handle multi-byte varints for tag + tag = tag_byte & 0x7F + shift = 7 + while tag_byte & 0x80 and pos < len(data): + tag_byte = data[pos] + pos += 1 + tag |= (tag_byte & 0x7F) << shift + shift += 7 + + field_num = tag >> 3 + wire_type = tag & 0x07 + + if field_num == 0 or field_num > 100: + break + + if wire_type == WIRE_VARINT: + # Read varint value + val = 0 + shift = 0 + while pos < len(data): + byte = data[pos] + pos += 1 + val |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + entry['fields'][field_num] = ('varint', val) + + elif wire_type == WIRE_FIXED32: + raw = data[pos:pos + 4] + pos += 4 + try: + f = struct.unpack(' 60 else ''}") + print(f" All fields present: {sorted(fields.keys())}") + + # Show any extra fields + for fn, fv in sorted(fields.items()): + if fn not in (1, 2, 3): + print(f" Field {fn}: {fv}") + + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Debug tank entries") + parser.add_argument('-i', '--interface', required=True, help='Interface IP') + parser.add_argument('-t', '--time', type=int, default=5, help='Capture time (seconds)') + args = parser.parse_args() + + print(f"Capturing tank data for {args.time} seconds...") + + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(('', 2565)) + mreq = struct.pack("4s4s", socket.inet_aton("226.192.206.102"), socket.inet_aton(args.interface)) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + sock.settimeout(1.0) + + seen_sizes = set() + end_time = time.time() + args.time + + try: + while time.time() < end_time: + try: + data, _ = sock.recvfrom(65535) + # Only process each unique packet size once + if len(data) not in seen_sizes: + seen_sizes.add(len(data)) + scan_packet(data) + except socket.timeout: + continue + except KeyboardInterrupt: + pass + finally: + sock.close() + + print("\n\nDone.") + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/debug/tank_finder.py b/axiom-nmea/debug/tank_finder.py new file mode 100755 index 0000000..904820f --- /dev/null +++ b/axiom-nmea/debug/tank_finder.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +""" +Tank Finder - Scan all multicast groups for values matching expected tank levels. +""" + +import struct +import socket +import time +import threading +from typing import Dict, Any, Optional, List, Tuple + +WIRE_VARINT = 0 +WIRE_FIXED64 = 1 +WIRE_LENGTH = 2 +WIRE_FIXED32 = 5 + +HEADER_SIZE = 20 + +MULTICAST_GROUPS = [ + ("226.192.206.98", 2561), + ("226.192.206.99", 2562), + ("226.192.206.100", 2563), + ("226.192.206.101", 2564), + ("226.192.206.102", 2565), + ("226.192.219.0", 3221), + ("239.2.1.1", 2154), +] + +# Target values to find (tank levels) +TARGET_VALUES = [ + (66, 70), # ~68% fuel tank + (87, 91), # ~89% fuel tank + (0.66, 0.70), # Decimal range + (0.87, 0.91), # Decimal range +] + + +class ProtobufParser: + def __init__(self, data: bytes): + self.data = data + self.pos = 0 + + def read_varint(self) -> int: + result = 0 + shift = 0 + while self.pos < len(self.data): + byte = self.data[self.pos] + self.pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + return result + + def parse(self, path: str = "") -> List[Tuple[str, str, Any]]: + """Parse and return list of (path, type, value) for all fields.""" + results = [] + while self.pos < len(self.data): + try: + tag = self.read_varint() + field_num = tag >> 3 + wire_type = tag & 0x07 + + if field_num == 0 or field_num > 1000: + break + + field_path = f"{path}.{field_num}" if path else str(field_num) + + if wire_type == WIRE_VARINT: + value = self.read_varint() + results.append((field_path, "varint", value)) + elif wire_type == WIRE_FIXED64: + raw = self.data[self.pos:self.pos + 8] + self.pos += 8 + try: + d = struct.unpack(' bool: + """Check if value matches our target ranges.""" + for low, high in TARGET_VALUES: + if low <= val <= high: + return True + return False + + +def scan_packet(data: bytes, group: str, port: int): + """Scan a packet for target values.""" + if len(data) < HEADER_SIZE + 5: + return + + proto_data = data[HEADER_SIZE:] + parser = ProtobufParser(proto_data) + fields = parser.parse() + + matches = [] + for path, vtype, value in fields: + if isinstance(value, (int, float)) and is_target_value(value): + matches.append((path, vtype, value)) + + if matches: + print(f"\n{'='*60}") + print(f"MATCH on {group}:{port} (packet size: {len(data)})") + print(f"{'='*60}") + for path, vtype, value in matches: + print(f" Field {path} ({vtype}): {value}") + + # Show all Field 16 entries (tank data) for context + print(f"\nAll Field 16 (Tank) entries:") + tank_entries = {} + for path, vtype, value in fields: + if path.startswith("16."): + parts = path.split(".") + if len(parts) >= 2: + # Group by the implicit index (based on order seen) + entry_key = path # We'll group differently + tank_entries[path] = (vtype, value) + + # Parse Field 16 entries properly - group consecutive 16.x fields + current_tank = {} + tank_list = [] + last_field = 0 + for path, vtype, value in fields: + if path.startswith("16."): + subfield = int(path.split(".")[1]) + # If we see a field number <= last, it's a new tank entry + if subfield <= last_field and current_tank: + tank_list.append(current_tank) + current_tank = {} + current_tank[subfield] = (vtype, value) + last_field = subfield + if current_tank: + tank_list.append(current_tank) + + for i, tank in enumerate(tank_list): + tank_id = tank.get(1, (None, "?"))[1] + status = tank.get(2, (None, "?"))[1] + level = tank.get(3, (None, "?"))[1] + print(f" Tank #{tank_id}: level={level}%, status={status}") + + +class MulticastScanner: + def __init__(self, interface_ip: str): + self.interface_ip = interface_ip + self.running = False + self.lock = threading.Lock() + + def _create_socket(self, group: str, port: int): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, 'SO_REUSEPORT'): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind(('', port)) + mreq = struct.pack("4s4s", socket.inet_aton(group), socket.inet_aton(self.interface_ip)) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + sock.settimeout(1.0) + return sock + + def _listen(self, sock, group: str, port: int): + seen_sizes = set() + while self.running: + try: + data, _ = sock.recvfrom(65535) + # Only process each unique packet size once per group + size_key = len(data) + if size_key not in seen_sizes: + seen_sizes.add(size_key) + with self.lock: + scan_packet(data, group, port) + except socket.timeout: + continue + except: + pass + + def start(self): + self.running = True + threads = [] + for group, port in MULTICAST_GROUPS: + try: + sock = self._create_socket(group, port) + t = threading.Thread(target=self._listen, args=(sock, group, port), daemon=True) + t.start() + threads.append(t) + print(f"Scanning {group}:{port}") + except Exception as e: + print(f"Error on {group}:{port}: {e}") + return threads + + def stop(self): + self.running = False + + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Find tank level values in multicast data") + parser.add_argument('-i', '--interface', required=True, help='Interface IP') + parser.add_argument('-t', '--time', type=int, default=10, help='Scan duration (seconds)') + args = parser.parse_args() + + print(f"Scanning for values around 37-40 (or 0.37-0.40)...") + print(f"Will scan for {args.time} seconds\n") + + scanner = MulticastScanner(args.interface) + scanner.start() + + try: + time.sleep(args.time) + except KeyboardInterrupt: + pass + finally: + scanner.stop() + print("\nDone scanning") + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/debug/watch_field.py b/axiom-nmea/debug/watch_field.py new file mode 100644 index 0000000..ae7697a --- /dev/null +++ b/axiom-nmea/debug/watch_field.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +""" +Single Field Monitor - Watch a specific field across packets. + +Usage: + python3 watch_field.py -i 192.168.1.100 --field 7.1 + python3 watch_field.py --pcap capture.pcap --field 7.1 + python3 watch_field.py --pcap capture.pcap --field 13.4 # TWD +""" + +import struct +import socket +import time +import argparse +import threading +from datetime import datetime +from typing import Dict, Any, Optional, List + +WIRE_VARINT = 0 +WIRE_FIXED64 = 1 +WIRE_LENGTH = 2 +WIRE_FIXED32 = 5 + +HEADER_SIZE = 20 + +MULTICAST_GROUPS = [ + ("226.192.206.102", 2565), +] + + +class ProtobufParser: + def __init__(self, data: bytes): + self.data = data + self.pos = 0 + + def read_varint(self) -> int: + result = 0 + shift = 0 + while self.pos < len(self.data): + byte = self.data[self.pos] + self.pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + return result + + def parse(self) -> Dict[int, Any]: + fields = {} + while self.pos < len(self.data): + try: + tag = self.read_varint() + field_num = tag >> 3 + wire_type = tag & 0x07 + if field_num == 0 or field_num > 1000: + break + + if wire_type == WIRE_VARINT: + value = self.read_varint() + children = None + elif wire_type == WIRE_FIXED64: + value = self.data[self.pos:self.pos + 8] + self.pos += 8 + children = None + elif wire_type == WIRE_LENGTH: + length = self.read_varint() + value = self.data[self.pos:self.pos + length] + self.pos += length + try: + nested = ProtobufParser(value) + children = nested.parse() + if nested.pos < len(value) * 0.5: + children = None + except: + children = None + elif wire_type == WIRE_FIXED32: + value = self.data[self.pos:self.pos + 4] + self.pos += 4 + children = None + else: + break + + fields[field_num] = (wire_type, value, children) + except: + break + return fields + + +def get_field(fields: Dict, path: List[int]): + """Navigate to a specific field path like [7, 1] for field 7.1""" + current = fields + for i, field_num in enumerate(path): + if field_num not in current: + return None, None, None + wire_type, value, children = current[field_num] + if i == len(path) - 1: + return wire_type, value, children + if children is None: + return None, None, None + current = children + return None, None, None + + +def format_value(wire_type: int, value: Any) -> str: + """Format value with multiple interpretations.""" + results = [] + + if wire_type == WIRE_VARINT: + results.append(f"int: {value}") + + elif wire_type == WIRE_FIXED64: + try: + d = struct.unpack(' {d:.1f} ft = {d * 0.3048:.1f} m") + except: + pass + results.append(f"hex: {value.hex()}") + + elif wire_type == WIRE_FIXED32: + try: + f = struct.unpack(' as angle: {f * 57.2958:.1f}°") + if 0 < f < 100: + results.append(f" -> as m/s: {f * 1.94384:.1f} kts") + if 0 < f < 1000: + results.append(f" -> as depth: {f:.1f} ft = {f * 0.3048:.1f} m") + except: + pass + results.append(f"hex: {value.hex()}") + + elif wire_type == WIRE_LENGTH: + results.append(f"bytes[{len(value)}]: {value[:20].hex()}...") + + return " | ".join(results) if results else "?" + + +def read_pcap(filename: str): + packets = [] + with open(filename, 'rb') as f: + header = f.read(24) + magic = struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append((ts_sec + ts_usec / 1e6, pkt_data[payload_start:])) + return packets + + +class LiveListener: + def __init__(self, interface_ip: str): + self.interface_ip = interface_ip + self.running = False + self.packets = [] + self.lock = threading.Lock() + + def _create_socket(self, group: str, port: int): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, 'SO_REUSEPORT'): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind(('', port)) + mreq = struct.pack("4s4s", socket.inet_aton(group), socket.inet_aton(self.interface_ip)) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + sock.settimeout(1.0) + return sock + + def _listen(self, sock): + while self.running: + try: + data, _ = sock.recvfrom(65535) + if len(data) >= 200: + with self.lock: + self.packets.append((time.time(), data)) + # Keep last 100 + if len(self.packets) > 100: + self.packets = self.packets[-100:] + except socket.timeout: + continue + except: + pass + + def start(self): + self.running = True + for group, port in MULTICAST_GROUPS: + try: + sock = self._create_socket(group, port) + t = threading.Thread(target=self._listen, args=(sock,), daemon=True) + t.start() + except Exception as e: + print(f"Error: {e}") + + def get_latest(self): + with self.lock: + if self.packets: + return self.packets[-1] + return None, None + + def stop(self): + self.running = False + + +def main(): + parser = argparse.ArgumentParser(description="Watch a specific protobuf field") + parser.add_argument('-i', '--interface', help='Interface IP for live capture') + parser.add_argument('--pcap', help='Read from pcap file') + parser.add_argument('-f', '--field', required=True, help='Field path like "7.1" or "13.4"') + parser.add_argument('-n', '--count', type=int, default=20, help='Number of samples to show') + parser.add_argument('-t', '--interval', type=float, default=1.0, help='Seconds between samples (live)') + args = parser.parse_args() + + if not args.pcap and not args.interface: + parser.error("Either --interface or --pcap required") + + # Parse field path + field_path = [int(x) for x in args.field.split('.')] + field_str = '.'.join(str(x) for x in field_path) + + print(f"Watching Field {field_str}") + print("=" * 80) + + if args.pcap: + packets = read_pcap(args.pcap) + print(f"Loaded {len(packets)} packets from {args.pcap}\n") + + count = 0 + for ts, pkt in packets: + if len(pkt) < HEADER_SIZE + 20: + continue + + proto_data = pkt[HEADER_SIZE:] + parser = ProtobufParser(proto_data) + fields = parser.parse() + + wire_type, value, children = get_field(fields, field_path) + if wire_type is not None: + timestamp = datetime.fromtimestamp(ts).strftime("%H:%M:%S.%f")[:-3] + val_str = format_value(wire_type, value) + print(f"[{timestamp}] {len(pkt):4d}B | Field {field_str}: {val_str}") + count += 1 + if count >= args.count: + break + + if count == 0: + print(f"Field {field_str} not found in any packets") + + else: + listener = LiveListener(args.interface) + listener.start() + print(f"Listening... showing {args.count} samples\n") + + try: + count = 0 + last_ts = 0 + while count < args.count: + time.sleep(args.interval) + ts, pkt = listener.get_latest() + if pkt is None or ts == last_ts: + continue + last_ts = ts + + proto_data = pkt[HEADER_SIZE:] + parser = ProtobufParser(proto_data) + fields = parser.parse() + + wire_type, value, children = get_field(fields, field_path) + if wire_type is not None: + timestamp = datetime.fromtimestamp(ts).strftime("%H:%M:%S.%f")[:-3] + val_str = format_value(wire_type, value) + print(f"[{timestamp}] {len(pkt):4d}B | Field {field_str}: {val_str}") + count += 1 + else: + print(f"[{datetime.now().strftime('%H:%M:%S')}] Field {field_str} not found in {len(pkt)}B packet") + + except KeyboardInterrupt: + print("\nStopped") + finally: + listener.stop() + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/debug/wind_finder.py b/axiom-nmea/debug/wind_finder.py new file mode 100644 index 0000000..ebbfadd --- /dev/null +++ b/axiom-nmea/debug/wind_finder.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +""" +Diagnostic tool to find wind speed and direction values in Raymarine packets. +Searches for float values matching expected ranges. +""" + +import struct +import sys +from collections import defaultdict + +# Expected values +# Wind speed: 15-20 kts = 7.7-10.3 m/s +# Wind direction: 60-90 degrees = 1.05-1.57 radians + +EXPECTED_SPEED_MS_MIN = 7.0 +EXPECTED_SPEED_MS_MAX = 12.0 +EXPECTED_DIR_RAD_MIN = 1.0 +EXPECTED_DIR_RAD_MAX = 1.7 + +PCAP_MAGIC = 0xa1b2c3d4 + +def decode_float(data, offset): + if offset + 4 > len(data): + return None + try: + return struct.unpack(' len(data): + return None + try: + return struct.unpack(' 42 and pkt_data[12:14] == b'\x08\x00': + ip_header_len = (pkt_data[14] & 0x0F) * 4 + payload_start = 14 + ip_header_len + 8 + if payload_start < len(pkt_data): + packets.append(pkt_data[payload_start:]) + return packets + +def find_wind_candidates(packets): + """Find all float values that could be wind speed or direction.""" + + speed_candidates = defaultdict(list) # offset -> list of values + dir_candidates = defaultdict(list) + + for pkt_idx, data in enumerate(packets): + if len(data) < 100: + continue + + # Search for 32-bit floats + for offset in range(0x30, min(len(data) - 4, 0x300)): + val = decode_float(data, offset) + if val is None or val != val: # NaN check + continue + + # Check for wind speed range (m/s) + if EXPECTED_SPEED_MS_MIN <= val <= EXPECTED_SPEED_MS_MAX: + speed_candidates[offset].append((pkt_idx, val, len(data))) + + # Check for direction range (radians) + if EXPECTED_DIR_RAD_MIN <= val <= EXPECTED_DIR_RAD_MAX: + dir_candidates[offset].append((pkt_idx, val, len(data))) + + return speed_candidates, dir_candidates + +def main(): + filename = sys.argv[1] if len(sys.argv) > 1 else "raymarine_sample.pcap" + + print(f"Reading {filename}...") + packets = read_pcap(filename) + print(f"Loaded {len(packets)} packets\n") + + print(f"Searching for wind speed values ({EXPECTED_SPEED_MS_MIN}-{EXPECTED_SPEED_MS_MAX} m/s)") + print(f"Searching for wind direction values ({EXPECTED_DIR_RAD_MIN}-{EXPECTED_DIR_RAD_MAX} rad)\n") + + speed_candidates, dir_candidates = find_wind_candidates(packets) + + print("=" * 70) + print("WIND SPEED CANDIDATES (m/s)") + print("=" * 70) + + # Sort by number of occurrences + for offset in sorted(speed_candidates.keys(), key=lambda x: -len(speed_candidates[x]))[:15]: + hits = speed_candidates[offset] + values = [v for _, v, _ in hits] + pkt_sizes = set(s for _, _, s in hits) + avg_val = sum(values) / len(values) + avg_kts = avg_val * 1.94384 + print(f" Offset 0x{offset:04x}: {len(hits):4d} hits, avg {avg_val:.2f} m/s ({avg_kts:.1f} kts), sizes: {sorted(pkt_sizes)[:5]}") + + print("\n" + "=" * 70) + print("WIND DIRECTION CANDIDATES (radians)") + print("=" * 70) + + for offset in sorted(dir_candidates.keys(), key=lambda x: -len(dir_candidates[x]))[:15]: + hits = dir_candidates[offset] + values = [v for _, v, _ in hits] + pkt_sizes = set(s for _, _, s in hits) + avg_val = sum(values) / len(values) + avg_deg = avg_val * 57.2958 + print(f" Offset 0x{offset:04x}: {len(hits):4d} hits, avg {avg_val:.2f} rad ({avg_deg:.1f}°), sizes: {sorted(pkt_sizes)[:5]}") + + # Look for paired speed+direction at consecutive offsets + print("\n" + "=" * 70) + print("SPEED+DIRECTION PAIRS (4 bytes apart)") + print("=" * 70) + + for speed_offset in speed_candidates: + dir_offset = speed_offset + 4 # Next float + if dir_offset in dir_candidates: + speed_hits = len(speed_candidates[speed_offset]) + dir_hits = len(dir_candidates[dir_offset]) + if speed_hits > 5 and dir_hits > 5: + speed_vals = [v for _, v, _ in speed_candidates[speed_offset]] + dir_vals = [v for _, v, _ in dir_candidates[dir_offset]] + avg_speed = sum(speed_vals) / len(speed_vals) * 1.94384 + avg_dir = sum(dir_vals) / len(dir_vals) * 57.2958 + print(f" Speed @ 0x{speed_offset:04x} ({speed_hits} hits), Dir @ 0x{dir_offset:04x} ({dir_hits} hits)") + print(f" -> Avg: {avg_speed:.1f} kts @ {avg_dir:.1f}°") + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/examples/dbus-raymarine-publisher/README.md b/axiom-nmea/examples/dbus-raymarine-publisher/README.md new file mode 100644 index 0000000..47d27f2 --- /dev/null +++ b/axiom-nmea/examples/dbus-raymarine-publisher/README.md @@ -0,0 +1,487 @@ +# Raymarine D-Bus Publisher + +Publishes Raymarine sensor data to Venus OS via D-Bus, making it available to the Victron ecosystem (VRM, GX Touch display, etc.). Also runs an NMEA TCP server for navigation apps. + +## Published Services + +| Service | Description | +|---------|-------------| +| `com.victronenergy.gps.raymarine_0` | GPS position, speed, course | +| `com.victronenergy.meteo.raymarine_0` | Wind direction/speed, air temp, pressure | +| `com.victronenergy.navigation.raymarine_0` | Heading, depth, water temperature | +| `com.victronenergy.tank.raymarine_tankN_0` | Tank level for each tank | +| `com.victronenergy.battery.raymarine_batN_0` | Battery voltage for each battery | + +## Quick Deployment + +### 1. Build the package + +From the `examples/dbus-raymarine-publisher` directory: + +```bash +./build-package.sh +``` + +This creates `dbus-raymarine-publisher-1.0.0.tar.gz` with the following structure: +``` +dbus-raymarine-publisher/ + raymarine_nmea/ # Python library + venus_publisher.py # Publisher script + service/ # Daemontools service files + install.sh # Installation script + uninstall.sh # Removal script + README.md # This file + VERSION # Build info +``` + +### 2. Copy to Venus OS + +```bash +scp dbus-raymarine-publisher-1.0.0.tar.gz root@venus:/data/ +``` + +### 3. Extract on Venus OS + +```bash +ssh root@venus +cd /data +tar -xzf dbus-raymarine-publisher-1.0.0.tar.gz +``` + +### 4. Run the installer + +```bash +bash /data/dbus-raymarine-publisher/install.sh +``` + +The installer will: +- Find velib_python and create a symlink +- Prompt you to select a network interface (eth0, wlan0, or custom IP) +- Configure the NMEA TCP server port +- Install the daemontools service +- Set up automatic logging +- Configure rc.local for firmware update survival + +### 4b. Or test manually first + +```bash +cd /data/dbus-raymarine-publisher +python3 venus_publisher.py --interface eth0 +``` + +## Service Management + +After installation, control the service with: + +```bash +# Check status +svstat /service/dbus-raymarine-publisher + +# View logs +tail -F /var/log/dbus-raymarine-publisher/current | tai64nlocal + +# Stop service +svc -d /service/dbus-raymarine-publisher + +# Start service +svc -u /service/dbus-raymarine-publisher + +# Restart service +svc -t /service/dbus-raymarine-publisher +``` + +## Changing the Network Interface + +Edit the run script: +```bash +vi /data/dbus-raymarine-publisher/service/run +``` + +Change the `INTERFACE` variable to `eth0`, `wlan0`, or a specific IP address. + +Then restart the service: +```bash +svc -t /service/dbus-raymarine-publisher +``` + +## Surviving Firmware Updates + +The installer automatically configures `/data/rc.local` to restore the service symlink after Venus OS firmware updates. + +## Uninstalling + +```bash +bash /data/dbus-raymarine-publisher/uninstall.sh +``` + +To completely remove all files: +```bash +rm -rf /data/dbus-raymarine-publisher /var/log/dbus-raymarine-publisher +``` + +## Command Line Options + +| Option | Description | +|--------|-------------| +| `--interface IP` | Network interface or IP (default: 198.18.5.5) | +| `--no-gps` | Disable GPS service | +| `--no-meteo` | Disable Meteo (wind) service | +| `--no-navigation` | Disable Navigation service (heading, depth, water temp) | +| `--no-tanks` | Disable all Tank services | +| `--no-batteries` | Disable all Battery services | +| `--tank-ids 1,2,10` | Only publish specific tanks | +| `--battery-ids 11,13` | Only publish specific batteries | +| `--update-interval MS` | D-Bus update interval (default: 1000ms) | +| `--nmea-tcp-port PORT` | NMEA TCP server port (default: 10110) | +| `--no-nmea-tcp` | Disable NMEA TCP server | +| `--debug` | Enable debug logging | +| `--dry-run` | Listen without D-Bus registration | + +## D-Bus Path Reference + +### GPS (`com.victronenergy.gps`) + +| Path | Description | Unit | +|------|-------------|------| +| `/Position/Latitude` | Latitude | degrees | +| `/Position/Longitude` | Longitude | degrees | +| `/Speed` | Speed over ground | m/s | +| `/Course` | Course over ground | degrees | +| `/Fix` | GPS fix status | 0=no fix, 1=fix | + +### Meteo (`com.victronenergy.meteo`) + +| Path | Description | Unit | +|------|-------------|------| +| `/WindDirection` | True wind direction | degrees | +| `/WindSpeed` | True wind speed | m/s | +| `/ExternalTemperature` | Air temperature | C | +| `/Pressure` | Barometric pressure | hPa | + +### Navigation (`com.victronenergy.navigation`) + +| Path | Description | Unit | +|------|-------------|------| +| `/Heading` | True heading | degrees | +| `/Depth` | Depth below transducer | m | +| `/WaterTemperature` | Water temperature | C | + +### Tank (`com.victronenergy.tank`) + +| Path | Description | Unit | +|------|-------------|------| +| `/Level` | Tank level | 0-100% | +| `/Remaining` | Remaining volume | m3 | +| `/Capacity` | Tank capacity | m3 | +| `/FluidType` | Fluid type | enum | +| `/Status` | Sensor status | 0=OK | + +Fluid types: 0=Fuel, 1=Fresh water, 2=Waste water, 3=Live well, 4=Oil, 5=Black water + +### Battery (`com.victronenergy.battery`) + +| Path | Description | Unit | +|------|-------------|------| +| `/Dc/0/Voltage` | Battery voltage | V | +| `/Soc` | State of charge (estimated) | % | +| `/Alarms/LowVoltage` | Low voltage alarm | 0/1/2 | +| `/Alarms/HighVoltage` | High voltage alarm | 0/1/2 | + +## Testing with dbus-spy + +On Venus OS, use `dbus-spy` to view published data: + +```bash +# List all Raymarine services +dbus -y | grep raymarine + +# Read GPS position +dbus -y com.victronenergy.gps.raymarine_0 /Position/Latitude GetValue +dbus -y com.victronenergy.gps.raymarine_0 /Position/Longitude GetValue + +# Read navigation data +dbus -y com.victronenergy.navigation.raymarine_0 /Heading GetValue +dbus -y com.victronenergy.navigation.raymarine_0 /Depth GetValue + +# Read wind data +dbus -y com.victronenergy.meteo.raymarine_0 /WindDirection GetValue +dbus -y com.victronenergy.meteo.raymarine_0 /WindSpeed GetValue + +# Read tank levels +dbus -y com.victronenergy.tank.raymarine_tank1_0 /Level GetValue +``` + +## Network Requirements + +The Venus OS device must be connected to the same network as the Raymarine LightHouse MFD. The service will use the selected interface (eth0 or wlan0) and resolve the IP address at runtime, which works with DHCP. + +If you need a specific VLAN IP (e.g., 198.18.x.x), you can add it manually: + +```bash +# Add VLAN interface (temporary) +ip addr add 198.18.4.108/16 dev eth0 + +# Or configure in /etc/network/interfaces for persistence +``` + +## MQTT Access + +Venus OS includes an MQTT broker that mirrors all D-Bus values, allowing external systems (Home Assistant, Node-RED, SignalK, etc.) to access sensor data. + +### Enabling MQTT + +On the GX device (or Venus OS): + +1. Go to **Settings > Services > MQTT** +2. Enable **MQTT on LAN** +3. Optionally enable **MQTT on LAN (SSL)** for encrypted connections + +Default ports: +- **1883** - MQTT (unencrypted) +- **8883** - MQTT with SSL + +### MQTT Topic Structure + +Venus OS uses this topic structure: + +``` +N//// +``` + +Where: +- `N/` - Notification topic (read values) +- `` - Unique VRM portal ID (e.g., `b827eb123456`) +- `` - Service category (gps, tank, battery, meteo, navigation) +- `` - Device instance number +- `` - D-Bus path without leading slash + +To write values, use `W/` prefix instead of `N/`. + +### Raymarine Sensor MQTT Topics + +#### GPS Topics + +| MQTT Topic | Description | Unit | +|------------|-------------|------| +| `N//gps/0/Position/Latitude` | Latitude | degrees | +| `N//gps/0/Position/Longitude` | Longitude | degrees | +| `N//gps/0/Speed` | Speed over ground | m/s | +| `N//gps/0/Course` | Course over ground | degrees | +| `N//gps/0/Fix` | GPS fix status | 0=no fix, 1=fix | + +#### Meteo/Wind Topics + +| MQTT Topic | Description | Unit | +|------------|-------------|------| +| `N//meteo/0/WindDirection` | True wind direction | degrees | +| `N//meteo/0/WindSpeed` | True wind speed | m/s | +| `N//meteo/0/ExternalTemperature` | Air temperature | C | +| `N//meteo/0/Pressure` | Barometric pressure | hPa | + +#### Navigation Topics + +| MQTT Topic | Description | Unit | +|------------|-------------|------| +| `N//navigation/0/Heading` | True heading | degrees | +| `N//navigation/0/Depth` | Depth below transducer | m | +| `N//navigation/0/WaterTemperature` | Water temperature | C | + +#### Tank Topics + +Each tank is published as a separate instance. With default tank configuration: + +| Tank | Instance | MQTT Base Topic | +|------|----------|-----------------| +| Fuel Starboard (ID 1) | 0 | `N//tank/0/...` | +| Fuel Port (ID 2) | 1 | `N//tank/1/...` | +| Water Bow (ID 10) | 2 | `N//tank/2/...` | +| Water Stern (ID 11) | 3 | `N//tank/3/...` | +| Black Water (ID 100) | 4 | `N//tank/4/...` | + +Available paths per tank instance: + +| Path Suffix | Description | Unit | +|-------------|-------------|------| +| `/Level` | Tank fill level | 0-100% | +| `/Remaining` | Remaining volume | m3 | +| `/Capacity` | Total capacity | m3 | +| `/FluidType` | Fluid type enum | see below | +| `/Status` | Sensor status | 0=OK | +| `/CustomName` | Tank name | string | + +Example full topics for Fuel Starboard tank: +``` +N//tank/0/Level +N//tank/0/Remaining +N//tank/0/Capacity +N//tank/0/FluidType +``` + +#### Battery Topics + +Each battery is published as a separate instance. With default battery configuration: + +| Battery | Instance | MQTT Base Topic | +|---------|----------|-----------------| +| House Bow (ID 11) | 0 | `N//battery/0/...` | +| House Stern (ID 13) | 1 | `N//battery/1/...` | +| Engine Port (ID 1000) | 2 | `N//battery/2/...` | +| Engine Starboard (ID 1001) | 3 | `N//battery/3/...` | + +Available paths per battery instance: + +| Path Suffix | Description | Unit | +|-------------|-------------|------| +| `/Dc/0/Voltage` | Battery voltage | V DC | +| `/Soc` | State of charge (estimated) | 0-100% | +| `/Alarms/LowVoltage` | Low voltage alarm | 0/1/2 | +| `/Alarms/HighVoltage` | High voltage alarm | 0/1/2 | +| `/CustomName` | Battery name | string | + +Example full topics for House Bow battery: +``` +N//battery/0/Dc/0/Voltage +N//battery/0/Soc +N//battery/0/Alarms/LowVoltage +``` + +### MQTT Message Format + +Values are published as JSON: + +```json +{"value": 25.4} +``` + +For string values: +```json +{"value": "Fuel Starboard"} +``` + +### Subscribing to All Raymarine Sensors + +```bash +# Subscribe to all GPS data +mosquitto_sub -h -t 'N/+/gps/#' -v + +# Subscribe to all navigation data +mosquitto_sub -h -t 'N/+/navigation/#' -v + +# Subscribe to all tank data +mosquitto_sub -h -t 'N/+/tank/#' -v + +# Subscribe to all battery data +mosquitto_sub -h -t 'N/+/battery/#' -v + +# Subscribe to all meteo/wind data +mosquitto_sub -h -t 'N/+/meteo/#' -v + +# Subscribe to everything +mosquitto_sub -h -t 'N/#' -v +``` + +### Keep-Alive Requirement + +Venus OS MQTT requires periodic keep-alive messages to continue receiving updates. Send an empty message to the read topic: + +```bash +# Initial subscription request +mosquitto_pub -h -t 'R//system/0/Serial' -m '' + +# Or request all values +mosquitto_pub -h -t 'R//keepalive' -m '' +``` + +For continuous monitoring, send keep-alive every 30-60 seconds. + +### Home Assistant Integration + +Example MQTT sensor configuration for Home Assistant: + +```yaml +mqtt: + sensor: + # GPS Position + - name: "Boat Latitude" + state_topic: "N//gps/0/Position/Latitude" + value_template: "{{ value_json.value }}" + unit_of_measurement: "deg" + + - name: "Boat Longitude" + state_topic: "N//gps/0/Position/Longitude" + value_template: "{{ value_json.value }}" + unit_of_measurement: "deg" + + - name: "Boat Speed" + state_topic: "N//gps/0/Speed" + value_template: "{{ (value_json.value * 1.94384) | round(1) }}" + unit_of_measurement: "kn" + + # Navigation + - name: "Boat Heading" + state_topic: "N//navigation/0/Heading" + value_template: "{{ value_json.value }}" + unit_of_measurement: "deg" + + - name: "Water Depth" + state_topic: "N//navigation/0/Depth" + value_template: "{{ (value_json.value * 3.28084) | round(1) }}" + unit_of_measurement: "ft" + + # Tank Levels + - name: "Fuel Starboard" + state_topic: "N//tank/0/Level" + value_template: "{{ value_json.value }}" + unit_of_measurement: "%" + device_class: battery + + - name: "Fresh Water Bow" + state_topic: "N//tank/2/Level" + value_template: "{{ value_json.value }}" + unit_of_measurement: "%" + + # Battery Voltage + - name: "House Battery Voltage" + state_topic: "N//battery/0/Dc/0/Voltage" + value_template: "{{ value_json.value | round(2) }}" + unit_of_measurement: "V" + device_class: voltage +``` + +### Node-RED Integration + +Example Node-RED flow to monitor tank levels: + +1. Add **mqtt in** node subscribed to `N//tank/+/Level` +2. Add **json** node to parse the message +3. Add **function** node: + ```javascript + msg.payload = msg.payload.value; + msg.topic = msg.topic.split('/')[3]; // Extract tank instance + return msg; + ``` +4. Connect to dashboard gauge or further processing + +### SignalK Integration + +SignalK can subscribe to Venus MQTT and convert values to SignalK paths. Install the `signalk-venus-plugin` for automatic integration, or manually map MQTT topics in the SignalK server configuration. + +## Troubleshooting + +### No D-Bus services registered +- Check that velib_python is installed (pre-installed on Venus OS) +- Ensure dbus-python with GLib support is available + +### No data received +- Verify network interface is configured correctly +- Check that the Raymarine MFD is broadcasting on the network +- Use `--debug` to see raw packet data + +### Stale data +- Data older than 10-30 seconds is marked as stale +- Check network connectivity to Raymarine multicast groups + +### Service won't start +- Check logs: `tail /var/log/dbus-raymarine-publisher/current` +- Verify run script is executable: `ls -la /service/dbus-raymarine-publisher/run` +- Check service status: `svstat /service/dbus-raymarine-publisher` diff --git a/axiom-nmea/examples/dbus-raymarine-publisher/build-package.sh b/axiom-nmea/examples/dbus-raymarine-publisher/build-package.sh new file mode 100755 index 0000000..90c52f4 --- /dev/null +++ b/axiom-nmea/examples/dbus-raymarine-publisher/build-package.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# +# Build script for Raymarine D-Bus Publisher Venus OS package +# +# Creates a tar.gz package that can be: +# 1. Copied to a Venus OS device (Cerbo GX, Venus GX, etc.) +# 2. Untarred to /data/ +# 3. Installed by running install.sh +# +# Usage: +# ./build-package.sh # Creates package with default name +# ./build-package.sh --version 1.0.0 # Creates package with version in name +# ./build-package.sh --output /path/ # Specify output directory +# +# Installation on Venus OS: +# scp dbus-raymarine-publisher-*.tar.gz root@:/data/ +# ssh root@ +# cd /data && tar -xzf dbus-raymarine-publisher-*.tar.gz +# bash /data/dbus-raymarine-publisher/install.sh +# + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +VERSION="1.0.0" +OUTPUT_DIR="$SCRIPT_DIR" +PACKAGE_NAME="dbus-raymarine-publisher" + +while [[ $# -gt 0 ]]; do + case $1 in + --version|-v) + VERSION="$2" + shift 2 + ;; + --output|-o) + OUTPUT_DIR="$2" + shift 2 + ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " -v, --version VERSION Set package version (default: 1.0.0)" + echo " -o, --output PATH Output directory (default: script directory)" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +BUILD_DATE=$(date -u +"%Y-%m-%d %H:%M:%S UTC") +BUILD_TIMESTAMP=$(date +%Y%m%d%H%M%S) + +BUILD_DIR=$(mktemp -d) +PACKAGE_DIR="$BUILD_DIR/$PACKAGE_NAME" + +echo "==================================================" +echo "Building $PACKAGE_NAME package" +echo "==================================================" +echo "Version: $VERSION" +echo "Build date: $BUILD_DATE" +echo "Source: $SCRIPT_DIR" +echo "Output: $OUTPUT_DIR" +echo "" + +echo "1. Creating package structure..." +mkdir -p "$PACKAGE_DIR" +mkdir -p "$PACKAGE_DIR/service/log" + +[ "$(uname)" = "Darwin" ] && export COPYFILE_DISABLE=1 + +echo "2. Copying application files..." +cp "$SCRIPT_DIR/venus_publisher.py" "$PACKAGE_DIR/" + +echo "3. Copying raymarine_nmea library..." +cp -r "$PROJECT_ROOT/raymarine_nmea" "$PACKAGE_DIR/" + +echo "4. Copying service files..." +cp "$SCRIPT_DIR/service/run" "$PACKAGE_DIR/service/" +cp "$SCRIPT_DIR/service/log/run" "$PACKAGE_DIR/service/log/" + +echo "5. Copying installation scripts..." +cp "$SCRIPT_DIR/install.sh" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/uninstall.sh" "$PACKAGE_DIR/" + +echo "6. Copying documentation..." +if [ -f "$SCRIPT_DIR/README.md" ]; then + cp "$SCRIPT_DIR/README.md" "$PACKAGE_DIR/" +fi + +echo "7. Creating version info..." +cat > "$PACKAGE_DIR/VERSION" << EOF +Package: $PACKAGE_NAME +Version: $VERSION +Build Date: $BUILD_DATE +Build Timestamp: $BUILD_TIMESTAMP + +Installation: + 1. Copy to Venus OS: scp $PACKAGE_NAME-$VERSION.tar.gz root@:/data/ + 2. SSH to device: ssh root@ + 3. Extract: cd /data && tar -xzf $PACKAGE_NAME-$VERSION.tar.gz + 4. Install: bash /data/$PACKAGE_NAME/install.sh +EOF + +echo "8. Setting permissions..." +chmod +x "$PACKAGE_DIR/venus_publisher.py" +chmod +x "$PACKAGE_DIR/install.sh" +chmod +x "$PACKAGE_DIR/uninstall.sh" +chmod +x "$PACKAGE_DIR/service/run" +chmod +x "$PACKAGE_DIR/service/log/run" + +mkdir -p "$OUTPUT_DIR" + +TARBALL_NAME="$PACKAGE_NAME-$VERSION.tar.gz" +OUTPUT_DIR_ABS="$(cd "$OUTPUT_DIR" && pwd)" +TARBALL_PATH="$OUTPUT_DIR_ABS/$TARBALL_NAME" + +echo "9. Creating package archive..." +cd "$BUILD_DIR" +if [ "$(uname)" = "Darwin" ]; then + if command -v xattr >/dev/null 2>&1; then + xattr -cr "$PACKAGE_NAME" + fi +fi +tar --format=ustar -czf "$TARBALL_PATH" "$PACKAGE_NAME" + +if command -v sha256sum >/dev/null 2>&1; then + CHECKSUM=$(sha256sum "$TARBALL_PATH" | cut -d' ' -f1) +else + CHECKSUM=$(shasum -a 256 "$TARBALL_PATH" | cut -d' ' -f1) +fi +echo "$CHECKSUM $TARBALL_NAME" > "$OUTPUT_DIR_ABS/$TARBALL_NAME.sha256" + +echo "10. Cleaning up..." +rm -rf "$BUILD_DIR" + +FILE_SIZE=$(du -h "$TARBALL_PATH" | cut -f1) + +echo "" +echo "==================================================" +echo "Build complete!" +echo "==================================================" +echo "" +echo "Package: $TARBALL_PATH" +echo "Size: $FILE_SIZE" +echo "SHA256: $CHECKSUM" +echo "" +echo "Installation on Venus OS:" +echo " scp $TARBALL_PATH root@:/data/" +echo " ssh root@" +echo " cd /data" +echo " tar -xzf $TARBALL_NAME" +echo " bash /data/$PACKAGE_NAME/install.sh" +echo "" diff --git a/axiom-nmea/examples/dbus-raymarine-publisher/install.sh b/axiom-nmea/examples/dbus-raymarine-publisher/install.sh new file mode 100755 index 0000000..f075f87 --- /dev/null +++ b/axiom-nmea/examples/dbus-raymarine-publisher/install.sh @@ -0,0 +1,237 @@ +#!/bin/bash +# +# Installation script for Raymarine D-Bus Publisher on Venus OS +# +# Run this on the Venus OS device after copying files to /data/dbus-raymarine-publisher/ +# +# Usage: +# chmod +x install.sh +# ./install.sh +# + +set -e + +INSTALL_DIR="/data/dbus-raymarine-publisher" +SERVICE_LINK="dbus-raymarine-publisher" + +# Find velib_python +VELIB_DIR="" +if [ -d "/opt/victronenergy/velib_python" ]; then + VELIB_DIR="/opt/victronenergy/velib_python" +else + for candidate in \ + "/opt/victronenergy/dbus-systemcalc-py/ext/velib_python" \ + "/opt/victronenergy/dbus-generator/ext/velib_python" \ + "/opt/victronenergy/dbus-mqtt/ext/velib_python" \ + "/opt/victronenergy/dbus-digitalinputs/ext/velib_python" \ + "/opt/victronenergy/vrmlogger/ext/velib_python" + do + if [ -d "$candidate" ] && [ -f "$candidate/vedbus.py" ]; then + VELIB_DIR="$candidate" + break + fi + done +fi + +if [ -z "$VELIB_DIR" ]; then + VEDBUS_PATH=$(find /opt/victronenergy -name "vedbus.py" -path "*/velib_python/*" 2>/dev/null | head -1) + if [ -n "$VEDBUS_PATH" ]; then + VELIB_DIR=$(dirname "$VEDBUS_PATH") + fi +fi + +# Determine service directory +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +elif [ -L "/service" ]; then + SERVICE_DIR=$(readlink -f /service) +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "==================================================" +echo "Raymarine D-Bus Publisher - Installation" +echo "==================================================" + +if [ ! -d "$SERVICE_DIR" ]; then + echo "ERROR: This doesn't appear to be a Venus OS device." + echo " Service directory not found." + exit 1 +fi + +echo "Detected service directory: $SERVICE_DIR" + +if [ ! -f "$INSTALL_DIR/venus_publisher.py" ]; then + echo "ERROR: Installation files not found in $INSTALL_DIR" + echo " Please copy all files to $INSTALL_DIR first." + exit 1 +fi +if [ ! -f "$INSTALL_DIR/service/run" ]; then + echo "ERROR: service/run not found. The package is incomplete." + exit 1 +fi + +echo "1. Making scripts executable..." +chmod +x "$INSTALL_DIR/service/run" +chmod +x "$INSTALL_DIR/service/log/run" +chmod +x "$INSTALL_DIR/venus_publisher.py" + +echo "2. Creating velib_python symlink..." +if [ -z "$VELIB_DIR" ]; then + echo "ERROR: Could not find velib_python on this system." + exit 1 +fi +echo " Found velib_python at: $VELIB_DIR" +mkdir -p "$INSTALL_DIR/ext" +if [ -L "$INSTALL_DIR/ext/velib_python" ]; then + CURRENT_TARGET=$(readlink "$INSTALL_DIR/ext/velib_python") + if [ "$CURRENT_TARGET" != "$VELIB_DIR" ]; then + echo " Updating symlink (was: $CURRENT_TARGET)" + rm "$INSTALL_DIR/ext/velib_python" + fi +fi +if [ ! -L "$INSTALL_DIR/ext/velib_python" ]; then + ln -s "$VELIB_DIR" "$INSTALL_DIR/ext/velib_python" + echo " Symlink created: $INSTALL_DIR/ext/velib_python -> $VELIB_DIR" +else + echo " Symlink already exists" +fi + +echo "3. Configuring network interface..." +echo "" +echo " Available network interfaces:" +for iface in $(ls /sys/class/net/ 2>/dev/null); do + ip=$(ip -4 addr show "$iface" 2>/dev/null | grep 'inet ' | awk '{print $2}' | cut -d/ -f1 | head -n 1) + if [ -n "$ip" ]; then + echo " $iface: $ip" + fi +done +echo "" +echo " Select network interface for Raymarine VLAN:" +echo " 1) eth0 - Ethernet (recommended)" +echo " 2) wlan0 - WiFi" +echo " 3) Enter a specific IP address" +echo "" +read -p " Choose [1-3]: " -n 1 -r CHOICE +echo "" + +case $CHOICE in + 1) INTERFACE="eth0" ;; + 2) INTERFACE="wlan0" ;; + 3) + read -p " Enter IP address: " INTERFACE + if ! echo "$INTERFACE" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'; then + echo " Invalid IP address. Using eth0." + INTERFACE="eth0" + fi + ;; + *) INTERFACE="eth0" ;; +esac + +echo " Using interface: $INTERFACE" + +echo "4. Configuring NMEA TCP server..." +echo "" +echo " The service can run an NMEA 0183 TCP server for navigation apps" +echo " (Navionics, iSailor, OpenCPN, SignalK, etc.)" +echo "" +echo " 1) Enable on default port 10110 (recommended)" +echo " 2) Enable on custom port" +echo " 3) Disable NMEA TCP server" +echo "" +read -p " Choose [1-3]: " -n 1 -r TCP_CHOICE +echo "" + +case $TCP_CHOICE in + 1) NMEA_TCP_PORT="10110" ;; + 2) + read -p " Enter TCP port (1024-65535): " NMEA_TCP_PORT + if ! echo "$NMEA_TCP_PORT" | grep -qE '^[0-9]+$' || [ "$NMEA_TCP_PORT" -lt 1024 ] || [ "$NMEA_TCP_PORT" -gt 65535 ]; then + echo " Invalid port. Using default 10110." + NMEA_TCP_PORT="10110" + fi + ;; + 3) NMEA_TCP_PORT="disabled" ;; + *) NMEA_TCP_PORT="10110" ;; +esac + +sed -i "s/^INTERFACE=.*/INTERFACE=\"$INTERFACE\"/" "$INSTALL_DIR/service/run" +sed -i "s/^NMEA_TCP_PORT=.*/NMEA_TCP_PORT=\"$NMEA_TCP_PORT\"/" "$INSTALL_DIR/service/run" + +echo "5. Creating service symlink..." +if [ -L "$SERVICE_DIR/$SERVICE_LINK" ]; then + echo " Service link already exists, removing old link..." + rm "$SERVICE_DIR/$SERVICE_LINK" +fi +if [ -e "$SERVICE_DIR/$SERVICE_LINK" ]; then + rm -rf "$SERVICE_DIR/$SERVICE_LINK" +fi +ln -s "$INSTALL_DIR/service" "$SERVICE_DIR/$SERVICE_LINK" + +if [ -L "$SERVICE_DIR/$SERVICE_LINK" ]; then + echo " Symlink created: $SERVICE_DIR/$SERVICE_LINK -> $INSTALL_DIR/service" +else + echo "ERROR: Failed to create service symlink" + exit 1 +fi + +echo "6. Creating log directory..." +mkdir -p /var/log/dbus-raymarine-publisher + +echo "7. Setting up rc.local for persistence..." +RC_LOCAL="/data/rc.local" +if [ ! -f "$RC_LOCAL" ]; then + echo "#!/bin/bash" > "$RC_LOCAL" + chmod +x "$RC_LOCAL" +fi + +if ! grep -q "dbus-raymarine-publisher" "$RC_LOCAL"; then + echo "" >> "$RC_LOCAL" + echo "# Raymarine D-Bus Publisher" >> "$RC_LOCAL" + echo "if [ ! -L $SERVICE_DIR/$SERVICE_LINK ]; then" >> "$RC_LOCAL" + echo " ln -s /data/dbus-raymarine-publisher/service $SERVICE_DIR/$SERVICE_LINK" >> "$RC_LOCAL" + echo "fi" >> "$RC_LOCAL" + echo " Added to rc.local for persistence across firmware updates" +else + echo " Already in rc.local" +fi + +echo "8. Activating service..." +sleep 2 +if command -v svstat >/dev/null 2>&1; then + if svstat "$SERVICE_DIR/$SERVICE_LINK" 2>/dev/null | grep -q "up"; then + echo " Service is running" + else + echo " Waiting for service to start..." + sleep 3 + fi +fi + +echo "" +echo "==================================================" +echo "Installation complete!" +echo "==================================================" +echo "" + +if command -v svstat >/dev/null 2>&1; then + echo "Current status:" + svstat "$SERVICE_DIR/$SERVICE_LINK" 2>/dev/null || echo " Service not yet detected by svscan" + echo "" +fi + +echo "Configuration:" +echo " Interface: $INTERFACE" +if [ "$NMEA_TCP_PORT" != "disabled" ]; then + echo " NMEA TCP port: $NMEA_TCP_PORT" +else + echo " NMEA TCP: disabled" +fi +echo "" +echo "To check status:" +echo " svstat $SERVICE_DIR/$SERVICE_LINK" +echo "" +echo "To view logs:" +echo " tail -F /var/log/dbus-raymarine-publisher/current | tai64nlocal" +echo "" diff --git a/axiom-nmea/examples/dbus-raymarine-publisher/service/log/run b/axiom-nmea/examples/dbus-raymarine-publisher/service/log/run new file mode 100755 index 0000000..e92f075 --- /dev/null +++ b/axiom-nmea/examples/dbus-raymarine-publisher/service/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec multilog t s99999 n8 /var/log/dbus-raymarine-publisher diff --git a/axiom-nmea/examples/dbus-raymarine-publisher/service/run b/axiom-nmea/examples/dbus-raymarine-publisher/service/run new file mode 100755 index 0000000..42b9437 --- /dev/null +++ b/axiom-nmea/examples/dbus-raymarine-publisher/service/run @@ -0,0 +1,34 @@ +#!/bin/sh +exec 2>&1 + +# Configuration - set by install.sh or edit manually +# Interface: network interface name (eth0, wlan0) or specific IP address +INTERFACE="eth0" +# NMEA TCP server port, or "disabled" to turn off +NMEA_TCP_PORT="10110" + +INSTALL_DIR="/data/dbus-raymarine-publisher" + +# Resolve interface name to IP address if needed +if echo "$INTERFACE" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'; then + INTERFACE_IP="$INTERFACE" +else + INTERFACE_IP=$(ip -4 addr show "$INTERFACE" 2>/dev/null | grep 'inet ' | awk '{print $2}' | cut -d/ -f1 | head -n 1) + if [ -z "$INTERFACE_IP" ]; then + echo "Error: Could not get IP address for interface $INTERFACE" + sleep 10 + exit 1 + fi +fi + +cd "$INSTALL_DIR" +export PYTHONPATH="$INSTALL_DIR/ext/velib_python:$PYTHONPATH" + +CMD_ARGS="--interface $INTERFACE_IP" +if [ -n "$NMEA_TCP_PORT" ] && [ "$NMEA_TCP_PORT" != "disabled" ]; then + CMD_ARGS="$CMD_ARGS --nmea-tcp-port $NMEA_TCP_PORT" +else + CMD_ARGS="$CMD_ARGS --no-nmea-tcp" +fi + +exec python3 "$INSTALL_DIR/venus_publisher.py" $CMD_ARGS diff --git a/axiom-nmea/examples/dbus-raymarine-publisher/uninstall.sh b/axiom-nmea/examples/dbus-raymarine-publisher/uninstall.sh new file mode 100755 index 0000000..99f8835 --- /dev/null +++ b/axiom-nmea/examples/dbus-raymarine-publisher/uninstall.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Uninstall Raymarine D-Bus Publisher for Venus OS + +INSTALL_DIR="/data/dbus-raymarine-publisher" +SERVICE_LINK="dbus-raymarine-publisher" + +# Find service directory +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "Uninstalling Raymarine D-Bus Publisher..." + +# Stop and remove service +if [ -L "$SERVICE_DIR/$SERVICE_LINK" ] || [ -e "$SERVICE_DIR/$SERVICE_LINK" ]; then + echo "Stopping and removing service..." + svc -d "$SERVICE_DIR/$SERVICE_LINK" 2>/dev/null || true + rm -f "$SERVICE_DIR/$SERVICE_LINK" + rm -rf "$SERVICE_DIR/$SERVICE_LINK" +fi + +echo "Service removed. Config and data in $INSTALL_DIR are preserved." +echo "To remove everything: rm -rf $INSTALL_DIR /var/log/dbus-raymarine-publisher" diff --git a/axiom-nmea/examples/dbus-raymarine-publisher/venus_publisher.py b/axiom-nmea/examples/dbus-raymarine-publisher/venus_publisher.py new file mode 100644 index 0000000..49836bf --- /dev/null +++ b/axiom-nmea/examples/dbus-raymarine-publisher/venus_publisher.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +""" +Venus OS D-Bus Publisher for Raymarine Sensor Data. + +This script reads sensor data from Raymarine LightHouse multicast +and publishes it to Venus OS via D-Bus, making it available to +the Victron ecosystem. It also runs an NMEA TCP server on port 10110 +for integration with navigation apps and charting software. + +Published D-Bus services: + - com.victronenergy.gps.raymarine_0 + GPS position, speed, and course + + - com.victronenergy.meteo.raymarine_0 + Wind direction and speed, air temperature, barometric pressure + + - com.victronenergy.tank.raymarine_tank{N}_0 + Tank levels for each configured tank + + - com.victronenergy.battery.raymarine_bat{N}_0 + Battery voltage for each configured battery + + - com.victronenergy.navigation.raymarine_0 + Heading, depth, and water temperature + +NMEA TCP Server (port 10110): + The NMEA TCP server broadcasts ALL available NMEA 0183 sentences, + which includes more data than what Venus OS can display via D-Bus: + - GPS: GGA, GLL, RMC + - Navigation: HDG, HDT, VTG, VHW + - Wind: MWV (apparent & true), MWD + - Depth: DPT, DBT + - Temperature: MTW, MTA + - Transducers: XDR (tanks, batteries, pressure) + + Compatible with Navionics, iSailor, OpenCPN, SignalK, and other + NMEA 0183 TCP clients. + +Usage: + # Basic usage (listens on 198.18.5.5 VLAN interface) + python3 venus_publisher.py + + # Specify interface IP + python3 venus_publisher.py --interface 198.18.5.5 + + # Enable debug logging + python3 venus_publisher.py --debug + + # Disable specific services + python3 venus_publisher.py --no-tanks --no-batteries + + # Use a different NMEA TCP port + python3 venus_publisher.py --nmea-tcp-port 2000 + + # Disable NMEA TCP server (D-Bus only) + python3 venus_publisher.py --no-nmea-tcp + +Installation on Venus OS: + 1. Build: ./build-package.sh + 2. Copy to device: scp dbus-raymarine-publisher-*.tar.gz root@:/data/ + 3. Extract: cd /data && tar -xzf dbus-raymarine-publisher-*.tar.gz + 4. Install: bash /data/dbus-raymarine-publisher/install.sh + +Requirements: + - Venus OS (or GLib + dbus-python for testing) + - Network access to Raymarine LightHouse multicast (VLAN interface) + - raymarine-nmea library + +Testing without Venus OS: + The script will start but D-Bus services won't register without + velib_python. Use --dry-run to test the listener without D-Bus. + +Author: Axiom NMEA Project +License: MIT +""" + +import argparse +import logging +import sys +import os +import time + +# Add script directory to path (raymarine_nmea is bundled alongside this script +# in deployed packages, or two levels up in the source tree) +_script_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, _script_dir) +sys.path.insert(0, os.path.dirname(os.path.dirname(_script_dir))) + +from raymarine_nmea import ( + RaymarineDecoder, + SensorData, + MulticastListener, + NMEATcpServer, + TANK_CONFIG, + BATTERY_CONFIG, +) +from raymarine_nmea.venus_dbus import VenusPublisher + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler(sys.stdout), + ] +) +logger = logging.getLogger(__name__) + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description='Publish Raymarine sensor data to Venus OS D-Bus', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + + # Network settings + parser.add_argument( + '--interface', '-i', + default='198.18.5.5', + help='VLAN interface IP for Raymarine multicast (default: 198.18.5.5)', + ) + + # Service enable/disable + parser.add_argument( + '--no-gps', + action='store_true', + help='Disable GPS service', + ) + parser.add_argument( + '--no-meteo', + action='store_true', + help='Disable Meteo (wind) service', + ) + parser.add_argument( + '--no-tanks', + action='store_true', + help='Disable Tank services', + ) + parser.add_argument( + '--no-batteries', + action='store_true', + help='Disable Battery services', + ) + parser.add_argument( + '--no-navigation', + action='store_true', + help='Disable Navigation service (heading, depth, water temp)', + ) + + # Specific IDs + parser.add_argument( + '--tank-ids', + type=str, + help='Comma-separated tank IDs to publish (default: all configured)', + ) + parser.add_argument( + '--battery-ids', + type=str, + help='Comma-separated battery IDs to publish (default: all configured)', + ) + + # Update interval + parser.add_argument( + '--update-interval', + type=int, + default=1000, + help='D-Bus update interval in milliseconds (default: 1000)', + ) + + # NMEA TCP server settings + parser.add_argument( + '--nmea-tcp-port', + type=int, + default=10110, + help='NMEA TCP server port (default: 10110)', + ) + parser.add_argument( + '--no-nmea-tcp', + action='store_true', + help='Disable NMEA TCP server', + ) + + # Debugging + parser.add_argument( + '--debug', + action='store_true', + help='Enable debug logging', + ) + parser.add_argument( + '--dry-run', + action='store_true', + help='Start listener but don\'t register D-Bus services', + ) + + return parser.parse_args() + + +def main(): + """Main entry point.""" + args = parse_args() + + # Configure logging level + if args.debug: + logging.getLogger().setLevel(logging.DEBUG) + logging.getLogger('raymarine_nmea').setLevel(logging.DEBUG) + + # Parse tank and battery IDs if specified + tank_ids = None + if args.tank_ids: + tank_ids = [int(x.strip()) for x in args.tank_ids.split(',')] + + battery_ids = None + if args.battery_ids: + battery_ids = [int(x.strip()) for x in args.battery_ids.split(',')] + + # Log configuration + logger.info("=" * 60) + logger.info("Venus OS D-Bus Publisher for Raymarine Sensor Data") + logger.info("=" * 60) + logger.info(f"Interface IP: {args.interface}") + logger.info(f"GPS enabled: {not args.no_gps}") + logger.info(f"Meteo enabled: {not args.no_meteo}") + logger.info(f"Navigation enabled: {not args.no_navigation}") + logger.info(f"Tanks enabled: {not args.no_tanks}") + if not args.no_tanks: + ids = tank_ids or list(TANK_CONFIG.keys()) + logger.info(f" Tank IDs: {ids}") + logger.info(f"Batteries enabled: {not args.no_batteries}") + if not args.no_batteries: + ids = battery_ids or list(BATTERY_CONFIG.keys()) + logger.info(f" Battery IDs: {ids}") + logger.info(f"Update interval: {args.update_interval}ms") + logger.info(f"NMEA TCP server enabled: {not args.no_nmea_tcp}") + if not args.no_nmea_tcp: + logger.info(f" NMEA TCP port: {args.nmea_tcp_port}") + logger.info("=" * 60) + + # Create components + decoder = RaymarineDecoder() + sensor_data = SensorData() + + # Callback to log decoded data in debug mode + def on_decode(decoded): + if args.debug and decoded.has_data(): + logger.debug(f"Decoded: lat={decoded.latitude}, lon={decoded.longitude}, " + f"twd={decoded.twd_deg}, tanks={decoded.tanks}, " + f"batteries={decoded.batteries}") + + # Start multicast listener + logger.info(f"Starting multicast listener on {args.interface}...") + listener = MulticastListener( + decoder=decoder, + sensor_data=sensor_data, + interface_ip=args.interface, + on_decode=on_decode if args.debug else None, + ) + listener.start() + logger.info("Multicast listener started") + + # Create NMEA TCP server (broadcasts all NMEA sentences, more than D-Bus) + nmea_tcp_server = None + if not args.no_nmea_tcp: + nmea_tcp_server = NMEATcpServer( + sensor_data=sensor_data, + port=args.nmea_tcp_port, + ) + if nmea_tcp_server.start(): + logger.info(f"NMEA TCP server started on port {args.nmea_tcp_port}") + else: + logger.warning("Failed to start NMEA TCP server, continuing without it") + nmea_tcp_server = None + + # Dry run mode - just listen and print data + if args.dry_run: + logger.info("Dry run mode - press Ctrl+C to stop") + try: + while True: + # Broadcast NMEA sentences if TCP server is running + if nmea_tcp_server: + nmea_tcp_server.broadcast() + + time.sleep(args.update_interval / 1000.0) + data = sensor_data.to_dict() + tcp_clients = nmea_tcp_server.client_count if nmea_tcp_server else 0 + logger.info(f"Position: {data['position']}") + logger.info(f"Wind: {data['wind']}") + logger.info(f"Tanks: {data['tanks']}") + logger.info(f"Batteries: {data['batteries']}") + logger.info(f"NMEA TCP clients: {tcp_clients}") + logger.info("-" * 40) + except KeyboardInterrupt: + logger.info("Stopping...") + if nmea_tcp_server: + nmea_tcp_server.stop() + listener.stop() + return + + # Create Venus publisher + logger.info("Creating Venus OS D-Bus publisher...") + publisher = VenusPublisher( + sensor_data=sensor_data, + enable_gps=not args.no_gps, + enable_meteo=not args.no_meteo, + enable_navigation=not args.no_navigation, + enable_tanks=not args.no_tanks, + enable_batteries=not args.no_batteries, + tank_ids=tank_ids, + battery_ids=battery_ids, + update_interval_ms=args.update_interval, + ) + + # Log service status + for service in publisher.services: + logger.info(f" Service: {service.service_name}") + + # Run publisher with integrated NMEA TCP broadcasting + try: + logger.info("Starting D-Bus publisher...") + _run_with_nmea_tcp(publisher, nmea_tcp_server, args.update_interval) + except RuntimeError as e: + logger.error(f"Failed to start publisher: {e}") + logger.info("Falling back to NMEA TCP only mode (no D-Bus)") + + # Fall back to just NMEA TCP broadcasting + try: + while True: + if nmea_tcp_server: + nmea_tcp_server.broadcast() + time.sleep(args.update_interval / 1000.0) + data = sensor_data.to_dict() + tcp_clients = nmea_tcp_server.client_count if nmea_tcp_server else 0 + logger.info(f"GPS: {data['position']}") + logger.info(f"Wind: {data['wind']}") + logger.info(f"Stats: {data['stats']}") + logger.info(f"NMEA TCP clients: {tcp_clients}") + except KeyboardInterrupt: + pass + finally: + logger.info("Stopping services...") + if nmea_tcp_server: + nmea_tcp_server.stop() + listener.stop() + logger.info("Shutdown complete") + + +def _run_with_nmea_tcp(publisher: VenusPublisher, nmea_tcp_server, update_interval_ms: int): + """Run the Venus publisher with integrated NMEA TCP broadcasting. + + This combines D-Bus publishing with NMEA TCP broadcasting in the same + GLib main loop, ensuring both are updated at the same interval. + + Args: + publisher: VenusPublisher instance + nmea_tcp_server: NMEATcpServer instance (or None to skip) + update_interval_ms: Update interval in milliseconds + """ + import signal + + # Try to import GLib + try: + from gi.repository import GLib + except ImportError: + raise RuntimeError( + "GLib is required to run VenusPublisher. " + "Install PyGObject or use --dry-run mode." + ) + + # Set up D-Bus main loop + try: + from dbus.mainloop.glib import DBusGMainLoop + DBusGMainLoop(set_as_default=True) + except ImportError: + raise RuntimeError( + "dbus-python with GLib support is required. " + "Install python3-dbus on Venus OS." + ) + + # Start D-Bus services + if not publisher.start(): + logger.error("Failed to start VenusPublisher") + return + + # Main loop reference for signal handler + mainloop = None + + # Set up signal handlers for graceful shutdown + def signal_handler(signum, frame): + logger.info(f"Received signal {signum}, stopping...") + if mainloop: + mainloop.quit() + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # Combined update callback: D-Bus + NMEA TCP + def update_callback(): + # Update D-Bus services + if not publisher.update(): + return False + + # Broadcast NMEA sentences to TCP clients + if nmea_tcp_server: + nmea_tcp_server.broadcast() + + return True + + # Set up periodic updates + GLib.timeout_add(update_interval_ms, update_callback) + + # Run main loop + logger.info("Publisher running, press Ctrl+C to stop") + mainloop = GLib.MainLoop() + + try: + mainloop.run() + except Exception as e: + logger.error(f"Main loop error: {e}") + finally: + publisher.stop() + + +if __name__ == '__main__': + main() diff --git a/axiom-nmea/examples/pcap-to-nmea/pcap_to_nmea.py b/axiom-nmea/examples/pcap-to-nmea/pcap_to_nmea.py new file mode 100644 index 0000000..43366eb --- /dev/null +++ b/axiom-nmea/examples/pcap-to-nmea/pcap_to_nmea.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +""" +PCAP to NMEA Conversion Example + +This example demonstrates how to use the raymarine_nmea library to +read Raymarine packets from a PCAP file and generate NMEA sentences. + +Useful for: +- Testing without a live Raymarine network +- Analyzing captured data +- Generating NMEA data for replay + +Usage: + python pcap_to_nmea.py capture.pcap + + # Output JSON instead of NMEA + python pcap_to_nmea.py capture.pcap --json + + # Output specific sentence types only + python pcap_to_nmea.py capture.pcap --sentences gga,mwd,mtw +""" + +import argparse +import json +import os +import sys +from pathlib import Path + +# Add repo root to path for development +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) + +from raymarine_nmea import ( + RaymarineDecoder, + SensorData, + NMEAGenerator, +) +from raymarine_nmea.listeners import PcapReader + + +def main(): + parser = argparse.ArgumentParser( + description="Convert Raymarine PCAP to NMEA sentences" + ) + parser.add_argument( + 'pcap', + help='Path to PCAP file' + ) + parser.add_argument( + '--json', + action='store_true', + help='Output JSON instead of NMEA' + ) + parser.add_argument( + '--sentences', + help='Comma-separated list of sentence types to output' + ) + parser.add_argument( + '-v', '--verbose', + action='store_true', + help='Verbose output (show decode progress)' + ) + + args = parser.parse_args() + + # Check file exists + if not os.path.exists(args.pcap): + print(f"Error: File not found: {args.pcap}") + sys.exit(1) + + # Read PCAP + print(f"Reading {args.pcap}...", file=sys.stderr) + reader = PcapReader(args.pcap) + print(f"Found {len(reader)} packets", file=sys.stderr) + + # Decode all packets + decoder = RaymarineDecoder(verbose=args.verbose) + data = SensorData() + + decoded_count = 0 + for packet in reader: + result = decoder.decode(packet) + if result.has_data(): + decoded_count += 1 + data.update(result) + + print(f"Decoded {decoded_count} packets with data", file=sys.stderr) + + # Output format + if args.json: + # JSON output + print(json.dumps(data.to_dict(), indent=2)) + else: + # NMEA output + generator = NMEAGenerator() + + # Filter sentences if specified + if args.sentences: + enabled = set(s.strip().lower() for s in args.sentences.split(',')) + generator.enabled = enabled + + sentences = generator.generate_all(data) + + if sentences: + print("\n# Generated NMEA sentences:", file=sys.stderr) + for sentence in sentences: + print(sentence, end='') + else: + print("No NMEA sentences generated (no valid data)", file=sys.stderr) + + # Summary + print("\n# Summary:", file=sys.stderr) + print(f"# Packets: {data.packet_count}", file=sys.stderr) + print(f"# Decoded: {data.decode_count}", file=sys.stderr) + + with data._lock: + if data.latitude: + print(f"# GPS: {data.latitude:.6f}, {data.longitude:.6f}", + file=sys.stderr) + if data.heading_deg: + print(f"# Heading: {data.heading_deg:.1f}°", file=sys.stderr) + if data.twd_deg: + print(f"# Wind: {data.tws_kts:.1f} kts @ {data.twd_deg:.1f}°", + file=sys.stderr) + if data.depth_m: + print(f"# Depth: {data.depth_m:.1f} m", file=sys.stderr) + if data.tanks: + print(f"# Tanks: {len(data.tanks)}", file=sys.stderr) + if data.batteries: + print(f"# Batteries: {len(data.batteries)}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/examples/quickstart/quickstart.py b/axiom-nmea/examples/quickstart/quickstart.py new file mode 100644 index 0000000..b569814 --- /dev/null +++ b/axiom-nmea/examples/quickstart/quickstart.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +""" +Raymarine NMEA Library - Quick Start Example + +This is a minimal example showing how to use the library. +""" + +import sys +import time +from pathlib import Path + +# Add repo root to path for development +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) + +from raymarine_nmea import ( + RaymarineDecoder, + SensorData, + NMEAGenerator, + MulticastListener, +) + + +def main(): + # Check for interface IP argument + if len(sys.argv) < 2: + print("Usage: python quickstart.py ") + print("Example: python quickstart.py 198.18.5.5") + sys.exit(1) + + interface_ip = sys.argv[1] + + # Create components + decoder = RaymarineDecoder() + data = SensorData() + generator = NMEAGenerator() + + # Start listening + listener = MulticastListener( + decoder=decoder, + sensor_data=data, + interface_ip=interface_ip, + ) + listener.start() + + print(f"Listening on {interface_ip}...") + print("Press Ctrl+C to stop\n") + + try: + while True: + # Generate all available NMEA sentences + sentences = generator.generate_all(data) + + # Print each sentence + for sentence in sentences: + print(sentence, end='') + + if sentences: + print() # Blank line between updates + + time.sleep(1) + + except KeyboardInterrupt: + print("\nStopping...") + finally: + listener.stop() + + # Print final summary + print("\nFinal sensor data:") + print(f" GPS: {data.latitude}, {data.longitude}") + print(f" Heading: {data.heading_deg}") + print(f" Wind: {data.tws_kts} kts @ {data.twd_deg}°") + print(f" Depth: {data.depth_m} m") + print(f" Water temp: {data.water_temp_c}°C") + print(f" Tanks: {data.tanks}") + print(f" Batteries: {data.batteries}") + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/examples/sensor-monitor/sensor_monitor.py b/axiom-nmea/examples/sensor-monitor/sensor_monitor.py new file mode 100644 index 0000000..fc800b4 --- /dev/null +++ b/axiom-nmea/examples/sensor-monitor/sensor_monitor.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +""" +Raymarine Sensor Update Monitor + +Displays real-time frequency of updates from each sensor type on the Raymarine network. +Useful for diagnosing gaps or inconsistent data delivery. + +Usage: + python sensor_monitor.py -i 198.18.5.5 +""" + +import argparse +import logging +import sys +import time +from datetime import datetime +from pathlib import Path + +# Add repo root to path for development +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) + +from raymarine_nmea import RaymarineDecoder, MulticastListener, SensorData + + +def main(): + parser = argparse.ArgumentParser(description="Monitor Raymarine sensor update frequency") + parser.add_argument('-i', '--interface', required=True, + help='Interface IP for Raymarine multicast (e.g., 198.18.5.5)') + parser.add_argument('--interval', type=float, default=1.0, + help='Display refresh interval in seconds (default: 1.0)') + parser.add_argument('--debug', action='store_true', + help='Enable debug logging to see raw packets') + + args = parser.parse_args() + + # Enable debug logging if requested + if args.debug: + logging.basicConfig( + level=logging.DEBUG, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + else: + logging.basicConfig(level=logging.WARNING) + + sensor_data = SensorData() + decoder = RaymarineDecoder() + + listener = MulticastListener( + decoder=decoder, + sensor_data=sensor_data, + interface_ip=args.interface, + ) + + print(f"Starting Raymarine sensor monitor on interface {args.interface}...") + if args.debug: + print("Debug mode enabled - check logs for packet details") + listener.start() + + # Track update times for each sensor + last_values = {} + update_counts = {} + last_update_time = {} + max_gaps = {} + start_time = time.time() + + sensor_fields = [ + ('GPS', 'gps', lambda d: (d.latitude, d.longitude)), + ('Heading', 'heading', lambda d: d.heading_deg), + ('COG', 'heading', lambda d: d.cog_deg), + ('SOG', 'heading', lambda d: d.sog_kts), + ('Depth', 'depth', lambda d: d.depth_m), + ('Water Temp', 'temp', lambda d: d.water_temp_c), + ('Air Temp', 'temp', lambda d: d.air_temp_c), + ('Wind (Apparent)', 'wind', lambda d: (d.awa_deg, d.aws_kts)), + ('Wind (True)', 'wind', lambda d: (d.twd_deg, d.tws_kts)), + ('Pressure', 'pressure', lambda d: d.pressure_mbar), + ('Tanks', 'tank', lambda d: dict(d.tanks) if d.tanks else None), + ('Batteries', 'battery', lambda d: dict(d.batteries) if d.batteries else None), + ] + + for name, _, _ in sensor_fields: + update_counts[name] = 0 + last_update_time[name] = None + max_gaps[name] = 0 + last_values[name] = None + + try: + while True: + if not args.debug: + # Clear screen only in non-debug mode + print("\033[2J\033[H", end="") + + print("=" * 80) + print(f"RAYMARINE SENSOR UPDATE MONITOR - {datetime.now().strftime('%H:%M:%S')}") + print("=" * 80) + + elapsed = time.time() - start_time + print(f"Monitoring for: {elapsed:.1f}s") + print(f"Packets: {sensor_data.packet_count} | Decoded: {sensor_data.decode_count}") + print() + + print(f"{'Sensor':<18} {'Value':<25} {'Age':>8} {'Count':>7} {'Avg':>8} {'MaxGap':>8}") + print("-" * 80) + + # Get age mapping (must be outside lock since get_age() also locks) + age_values = {} + for age_type in ['gps', 'heading', 'depth', 'temp', 'wind', 'pressure', 'tank', 'battery']: + age_values[age_type] = sensor_data.get_age(age_type) + + with sensor_data._lock: + for name, age_type, getter in sensor_fields: + try: + value = getter(sensor_data) + except Exception: + value = None + + # Check if value changed + if value != last_values[name] and value is not None: + now = time.time() + if last_update_time[name] is not None: + gap = now - last_update_time[name] + if gap > max_gaps[name]: + max_gaps[name] = gap + last_update_time[name] = now + update_counts[name] += 1 + last_values[name] = value + + # Get age (from pre-fetched values) + age = age_values.get(age_type) + + # Format value for display + if value is None: + val_str = "-" + elif isinstance(value, tuple): + parts = [f"{v:.1f}" if v is not None else "-" for v in value] + val_str = ", ".join(parts) + elif isinstance(value, dict): + val_str = f"{len(value)} items" + elif isinstance(value, float): + val_str = f"{value:.2f}" + else: + val_str = str(value)[:25] + + # Truncate value string + if len(val_str) > 25: + val_str = val_str[:22] + "..." + + # Color based on age + if age is None: + color = "\033[90m" # Gray + age_str = "-" + elif age > 5: + color = "\033[91m" # Red + age_str = f"{age:.1f}s" + elif age > 2: + color = "\033[93m" # Yellow + age_str = f"{age:.1f}s" + else: + color = "\033[92m" # Green + age_str = f"{age:.1f}s" + + reset = "\033[0m" + + count = update_counts[name] + avg_str = f"{count/elapsed:.2f}/s" if elapsed > 0 and count > 0 else "-" + max_gap_str = f"{max_gaps[name]:.1f}s" if max_gaps[name] > 0 else "-" + + print(f"{color}{name:<18} {val_str:<25} {age_str:>8} {count:>7} {avg_str:>8} {max_gap_str:>8}{reset}") + + print() + print("=" * 80) + print("Press Ctrl+C to exit") + if args.debug: + print("\n--- Debug output follows ---\n") + + time.sleep(args.interval) + + except KeyboardInterrupt: + print("\nStopping monitor...") + listener.stop() + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/nmea-server/.env.example b/axiom-nmea/nmea-server/.env.example new file mode 100644 index 0000000..d58beda --- /dev/null +++ b/axiom-nmea/nmea-server/.env.example @@ -0,0 +1,21 @@ +# NMEA Server Configuration +# Copy this file to .env and modify as needed + +# Required: IP address of the interface connected to Raymarine network +# This is where multicast data is received from +# Find your interface IP with: ip addr show | grep "inet " +RAYMARINE_INTERFACE=198.18.5.5 + +# IP address to bind NMEA server (default: 0.0.0.0 = all interfaces) +# Use a specific IP to expose NMEA on a different interface than Raymarine +# Example: Receive from Raymarine on 198.18.5.5, serve NMEA on 198.18.10.62 +NMEA_HOST=0.0.0.0 + +# NMEA TCP server port (default: 10110) +NMEA_PORT=10110 + +# Update interval in seconds (default: 1.0) +UPDATE_INTERVAL=1.0 + +# Logging level: DEBUG, INFO, WARNING, ERROR +LOG_LEVEL=INFO diff --git a/axiom-nmea/nmea-server/Dockerfile b/axiom-nmea/nmea-server/Dockerfile new file mode 100644 index 0000000..fcc1583 --- /dev/null +++ b/axiom-nmea/nmea-server/Dockerfile @@ -0,0 +1,36 @@ +FROM python:3.11-slim + +LABEL maintainer="Axiom NMEA Project" +LABEL description="NMEA TCP Server for Raymarine LightHouse protocol" + +# Set working directory +WORKDIR /app + +# Copy the package files for installation +COPY pyproject.toml /app/ +COPY raymarine_nmea/ /app/raymarine_nmea/ + +# Install the package +RUN pip install --no-cache-dir -e . + +# Copy the server +COPY nmea-server/server.py /app/ + +# Create non-root user for security +RUN useradd --create-home --shell /bin/bash nmea && \ + chown -R nmea:nmea /app + +USER nmea + +# Default environment variables +ENV RAYMARINE_INTERFACE="" +ENV NMEA_HOST=0.0.0.0 +ENV NMEA_PORT=10110 +ENV UPDATE_INTERVAL=1.0 +ENV LOG_LEVEL=INFO + +# Expose the NMEA port +EXPOSE 10110 + +# Run the server +ENTRYPOINT ["python", "-u", "/app/server.py"] diff --git a/axiom-nmea/nmea-server/docker-compose.yml b/axiom-nmea/nmea-server/docker-compose.yml new file mode 100644 index 0000000..425609f --- /dev/null +++ b/axiom-nmea/nmea-server/docker-compose.yml @@ -0,0 +1,52 @@ +services: + nmea-server: + build: + context: .. + dockerfile: nmea-server/Dockerfile + container_name: nmea-server + restart: unless-stopped + + # Host network mode required for multicast reception + network_mode: host + + # Network capabilities required for proper socket handling + cap_add: + - NET_ADMIN + - NET_RAW + + environment: + # Required: IP address of the interface connected to Raymarine network + # This is where multicast data is received from + # Find with: ip addr show | grep "inet " + RAYMARINE_INTERFACE: ${RAYMARINE_INTERFACE:?Set RAYMARINE_INTERFACE to your network interface IP} + + # IP address to bind NMEA server (default: 0.0.0.0 = all interfaces) + # Use a specific IP to expose NMEA on a different interface than Raymarine + NMEA_HOST: ${NMEA_HOST:-0.0.0.0} + + # NMEA TCP server port (default: 10110) + NMEA_PORT: ${NMEA_PORT:-10110} + + # NMEA UDP broadcast port (optional - set to enable UDP) + NMEA_UDP_PORT: ${NMEA_UDP_PORT:-} + + # NMEA UDP destination IP (default: 255.255.255.255 broadcast) + NMEA_UDP_DEST: ${NMEA_UDP_DEST:-255.255.255.255} + + # Update interval in seconds (default: 1.0) + UPDATE_INTERVAL: ${UPDATE_INTERVAL:-1.0} + + # Logging level: DEBUG, INFO, WARNING, ERROR + LOG_LEVEL: ${LOG_LEVEL:-INFO} + + # Note: ports mapping not used with host network mode + # The server listens on NMEA_PORT (default 10110) directly on host + + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + + # Ensure clean shutdown + stop_grace_period: 10s diff --git a/axiom-nmea/nmea-server/server.py b/axiom-nmea/nmea-server/server.py new file mode 100644 index 0000000..559f550 --- /dev/null +++ b/axiom-nmea/nmea-server/server.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +""" +NMEA Data Server - Docker Daemon + +Simple TCP/UDP server that broadcasts NMEA sentences from Raymarine data. +""" + +import argparse +import logging +import os +import signal +import socket +import sys +import threading +import time +from pathlib import Path +from typing import Dict, Optional + +# Add repo root to path for development (Docker installs via pip) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from raymarine_nmea import ( + RaymarineDecoder, + SensorData, + MulticastListener, + NMEAGenerator, +) + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' +) +logger = logging.getLogger(__name__) + + +class NMEAServer: + """TCP/UDP server for NMEA broadcast.""" + + def __init__( + self, + host: str = '0.0.0.0', + tcp_port: int = 10110, + udp_port: Optional[int] = None, + udp_dest: str = '255.255.255.255', + interval: float = 1.0 + ): + self.host = host + self.tcp_port = tcp_port + self.udp_port = udp_port + self.udp_dest = udp_dest + self.interval = interval + + self.sensor_data = SensorData() + # Enable all available NMEA sentences + self.generator = NMEAGenerator( + enabled_sentences={ + # GPS + 'gga', 'gll', 'rmc', + # Navigation + 'hdg', 'hdt', 'vtg', 'vhw', + # Wind + 'mwv_apparent', 'mwv_true', 'mwd', + # Depth + 'dpt', 'dbt', + # Temperature + 'mtw', 'mta', + # Transducers (tanks, batteries, pressure) + 'xdr_tanks', 'xdr_batteries', 'xdr_pressure', + } + ) + + self.server_socket: Optional[socket.socket] = None + self.udp_socket: Optional[socket.socket] = None + self.clients: Dict[socket.socket, str] = {} + self.clients_lock = threading.Lock() + self.running = False + + def accept_loop(self): + """Accept new TCP client connections.""" + while self.running: + try: + client, addr = self.server_socket.accept() + addr_str = f"{addr[0]}:{addr[1]}" + client.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + with self.clients_lock: + self.clients[client] = addr_str + count = len(self.clients) + + logger.info(f"TCP client connected: {addr_str} (total: {count})") + + except socket.timeout: + continue + except OSError as e: + if self.running: + logger.debug(f"Accept error: {e}") + + def broadcast(self, data: bytes): + """Send data to all TCP clients and UDP.""" + # TCP broadcast + with self.clients_lock: + dead = [] + for client, addr in list(self.clients.items()): + try: + client.sendall(data) + except Exception as e: + logger.info(f"TCP client {addr} error: {e}") + dead.append(client) + + for client in dead: + addr = self.clients.pop(client, "unknown") + try: + client.close() + except Exception: + pass + logger.info(f"TCP client {addr} removed (total: {len(self.clients)})") + + # UDP broadcast + if self.udp_socket and self.udp_port: + try: + self.udp_socket.sendto(data, (self.udp_dest, self.udp_port)) + except Exception as e: + logger.debug(f"UDP send error: {e}") + + def run(self, interface_ip: str): + """Run the server.""" + self.running = True + + # Start Raymarine listener + decoder = RaymarineDecoder() + listener = MulticastListener( + decoder=decoder, + sensor_data=self.sensor_data, + interface_ip=interface_ip, + ) + listener.start() + + # Create TCP server socket + self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.server_socket.settimeout(1.0) + self.server_socket.bind((self.host, self.tcp_port)) + self.server_socket.listen(5) + + # Create UDP socket if enabled + if self.udp_port: + self.udp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.udp_socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + logger.info(f"UDP broadcast enabled: {self.udp_dest}:{self.udp_port}") + + # Start TCP accept thread + accept_thread = threading.Thread(target=self.accept_loop, daemon=True) + accept_thread.start() + + logger.info(f"TCP server on {self.host}:{self.tcp_port}") + logger.info(f"Raymarine interface: {interface_ip}") + + last_log = 0 + + try: + while self.running: + sentences = self.generator.generate_all(self.sensor_data) + if sentences: + data = ''.join(sentences).encode('ascii') + self.broadcast(data) + for s in sentences: + logger.info(f"TX: {s.strip()}") + else: + logger.warning("No NMEA data this cycle") + + now = time.time() + if now - last_log >= 30: + with self.sensor_data._lock: + lat = self.sensor_data.latitude + lon = self.sensor_data.longitude + if lat and lon: + logger.info(f"GPS: {lat:.6f}, {lon:.6f} | TCP clients: {len(self.clients)}") + last_log = now + + time.sleep(self.interval) + + finally: + self.running = False + listener.stop() + self.server_socket.close() + if self.udp_socket: + self.udp_socket.close() + with self.clients_lock: + for c in self.clients: + try: + c.close() + except Exception: + pass + logger.info("Server stopped") + + +_server: Optional[NMEAServer] = None + + +def signal_handler(signum, frame): + if _server: + _server.running = False + + +def main(): + global _server + + parser = argparse.ArgumentParser(description="NMEA server") + parser.add_argument('-i', '--interface', default=os.environ.get('RAYMARINE_INTERFACE')) + parser.add_argument('-H', '--host', default=os.environ.get('NMEA_HOST', '0.0.0.0')) + parser.add_argument('-p', '--port', type=int, default=int(os.environ.get('NMEA_PORT', '10110')), + help='TCP port (default: 10110)') + parser.add_argument('--udp-port', type=int, default=os.environ.get('NMEA_UDP_PORT'), + help='UDP port for broadcast (optional)') + parser.add_argument('--udp-dest', default=os.environ.get('NMEA_UDP_DEST', '255.255.255.255'), + help='UDP destination IP (default: 255.255.255.255 broadcast)') + parser.add_argument('--interval', type=float, default=float(os.environ.get('UPDATE_INTERVAL', '1.0'))) + parser.add_argument('--log-level', default=os.environ.get('LOG_LEVEL', 'INFO')) + + args = parser.parse_args() + + if not args.interface: + parser.error("Interface IP required (-i or RAYMARINE_INTERFACE)") + + logging.getLogger().setLevel(getattr(logging, args.log_level)) + + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + + _server = NMEAServer( + host=args.host, + tcp_port=args.port, + udp_port=args.udp_port, + udp_dest=args.udp_dest, + interval=args.interval + ) + + try: + _server.run(args.interface) + except KeyboardInterrupt: + pass + except Exception as e: + logger.error(f"Fatal: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/axiom-nmea/pyproject.toml b/axiom-nmea/pyproject.toml new file mode 100644 index 0000000..bcac5be --- /dev/null +++ b/axiom-nmea/pyproject.toml @@ -0,0 +1,38 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "raymarine-nmea" +version = "1.0.0" +description = "Decode Raymarine LightHouse protocol data and convert to NMEA 0183 sentences" +readme = "README.md" +requires-python = ">=3.8" +license = {text = "MIT"} +authors = [ + {name = "Axiom NMEA Project"} +] +keywords = ["raymarine", "nmea", "marine", "navigation", "gps", "sailing"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering :: GIS", +] + +# No external dependencies - uses only Python standard library +dependencies = [] + +[project.urls] +Homepage = "https://github.com/terbonium/axiom-nmea" +Repository = "https://github.com/terbonium/axiom-nmea" + +[tool.setuptools.packages.find] +include = ["raymarine_nmea*"] diff --git a/axiom-nmea/raymarine_nmea/__init__.py b/axiom-nmea/raymarine_nmea/__init__.py new file mode 100644 index 0000000..c0d239c --- /dev/null +++ b/axiom-nmea/raymarine_nmea/__init__.py @@ -0,0 +1,100 @@ +""" +Raymarine NMEA Library + +A Python library for decoding Raymarine LightHouse protocol data +and converting it to standard NMEA 0183 sentences. + +Example usage: + from raymarine_nmea import RaymarineDecoder, NMEAGenerator + + # Create decoder and generator + decoder = RaymarineDecoder() + generator = NMEAGenerator() + + # Decode a packet (from multicast or PCAP) + sensor_data = decoder.decode(packet_bytes) + + # Generate NMEA sentences + sentences = generator.generate_all(sensor_data) + for sentence in sentences: + print(sentence) +""" + +__version__ = "1.0.0" +__author__ = "Axiom NMEA Project" + +# Core protocol components +from .protocol.parser import ProtobufParser +from .protocol.decoder import RaymarineDecoder +from .protocol.constants import ( + WIRE_VARINT, + WIRE_FIXED64, + WIRE_LENGTH, + WIRE_FIXED32, + RAD_TO_DEG, + MS_TO_KTS, + FEET_TO_M, + KELVIN_OFFSET, +) + +# Data models +from .data.store import SensorData + +# NMEA generation +from .nmea.generator import NMEAGenerator +from .nmea.sentence import NMEASentence +from .nmea.server import NMEATcpServer + +# Listeners +from .listeners.multicast import MulticastListener +from .listeners.pcap import PcapReader + +# Sensor configurations +from .sensors import ( + TANK_CONFIG, + BATTERY_CONFIG, + MULTICAST_GROUPS, + MULTICAST_GROUPS_ALL, +) + +# Venus OS D-Bus publishing (optional import - only works on Venus OS) +try: + from . import venus_dbus + HAS_VENUS_DBUS = True +except ImportError: + venus_dbus = None + HAS_VENUS_DBUS = False + +__all__ = [ + # Version + "__version__", + # Protocol + "ProtobufParser", + "RaymarineDecoder", + # Constants + "WIRE_VARINT", + "WIRE_FIXED64", + "WIRE_LENGTH", + "WIRE_FIXED32", + "RAD_TO_DEG", + "MS_TO_KTS", + "FEET_TO_M", + "KELVIN_OFFSET", + # Data + "SensorData", + # NMEA + "NMEAGenerator", + "NMEASentence", + "NMEATcpServer", + # Listeners + "MulticastListener", + "PcapReader", + # Config + "TANK_CONFIG", + "BATTERY_CONFIG", + "MULTICAST_GROUPS", + "MULTICAST_GROUPS_ALL", + # Venus OS + "venus_dbus", + "HAS_VENUS_DBUS", +] diff --git a/axiom-nmea/raymarine_nmea/data/__init__.py b/axiom-nmea/raymarine_nmea/data/__init__.py new file mode 100644 index 0000000..148c27a --- /dev/null +++ b/axiom-nmea/raymarine_nmea/data/__init__.py @@ -0,0 +1,7 @@ +""" +Data storage module. +""" + +from .store import SensorData + +__all__ = ["SensorData"] diff --git a/axiom-nmea/raymarine_nmea/data/store.py b/axiom-nmea/raymarine_nmea/data/store.py new file mode 100644 index 0000000..f44017f --- /dev/null +++ b/axiom-nmea/raymarine_nmea/data/store.py @@ -0,0 +1,327 @@ +""" +Thread-safe sensor data storage. + +This module provides a thread-safe container for aggregating sensor data +from multiple decoded packets over time. +""" + +import threading +import time +from dataclasses import dataclass, field +from datetime import datetime +from typing import Dict, Any, Optional + +from ..sensors import ( + get_tank_name, + get_tank_capacity, + get_battery_name, +) +from ..protocol.constants import FEET_TO_M + + +@dataclass +class SensorData: + """Thread-safe container for current sensor readings. + + This class aggregates data from multiple decoded packets and provides + thread-safe access for concurrent updates and reads. + + Example: + data = SensorData() + decoder = RaymarineDecoder() + + # Update from decoded packet + result = decoder.decode(packet) + data.update(result) + + # Read current values + print(f"GPS: {data.latitude}, {data.longitude}") + print(f"Heading: {data.heading_deg}°") + """ + + # Position + latitude: Optional[float] = None + longitude: Optional[float] = None + + # Navigation + heading_deg: Optional[float] = None + cog_deg: Optional[float] = None + sog_kts: Optional[float] = None + + # Wind + twd_deg: Optional[float] = None # True Wind Direction + tws_kts: Optional[float] = None # True Wind Speed + awa_deg: Optional[float] = None # Apparent Wind Angle + aws_kts: Optional[float] = None # Apparent Wind Speed + + # Depth (stored in meters internally) + depth_m: Optional[float] = None + + # Temperature + water_temp_c: Optional[float] = None + air_temp_c: Optional[float] = None + + # Barometric pressure (stored in mbar internally) + pressure_mbar: Optional[float] = None + + # Tanks: dict of tank_id -> level percentage + tanks: Dict[int, float] = field(default_factory=dict) + + # Batteries: dict of battery_id -> voltage + batteries: Dict[int, float] = field(default_factory=dict) + + # Timestamps for each data type (Unix timestamp) + gps_time: float = 0 + heading_time: float = 0 + wind_time: float = 0 + depth_time: float = 0 + temp_time: float = 0 + pressure_time: float = 0 + tank_time: float = 0 + battery_time: float = 0 + + # Statistics + packet_count: int = 0 + decode_count: int = 0 + start_time: float = field(default_factory=time.time) + + # Thread safety + _lock: threading.Lock = field(default_factory=threading.Lock) + + @property + def depth_ft(self) -> Optional[float]: + """Get depth in feet.""" + if self.depth_m is None: + return None + return self.depth_m / FEET_TO_M + + @property + def water_temp_f(self) -> Optional[float]: + """Get water temperature in Fahrenheit.""" + if self.water_temp_c is None: + return None + return self.water_temp_c * 9/5 + 32 + + @property + def air_temp_f(self) -> Optional[float]: + """Get air temperature in Fahrenheit.""" + if self.air_temp_c is None: + return None + return self.air_temp_c * 9/5 + 32 + + @property + def pressure_inhg(self) -> Optional[float]: + """Get barometric pressure in inches of mercury.""" + if self.pressure_mbar is None: + return None + return self.pressure_mbar * 0.02953 + + @property + def uptime(self) -> float: + """Get uptime in seconds.""" + return time.time() - self.start_time + + def update(self, decoded: 'DecodedData') -> None: + """Update sensor data from a decoded packet. + + Args: + decoded: DecodedData object from RaymarineDecoder.decode() + """ + # Import here to avoid circular import + from ..protocol.decoder import DecodedData + + now = time.time() + + with self._lock: + self.packet_count += 1 + + if decoded.has_data(): + self.decode_count += 1 + + # Update GPS + if decoded.latitude is not None and decoded.longitude is not None: + self.latitude = decoded.latitude + self.longitude = decoded.longitude + self.gps_time = now + + # Update heading + if decoded.heading_deg is not None: + self.heading_deg = decoded.heading_deg + self.heading_time = now + + # Update COG/SOG + if decoded.cog_deg is not None: + self.cog_deg = decoded.cog_deg + if decoded.sog_kts is not None: + self.sog_kts = decoded.sog_kts + + # Update wind + if (decoded.twd_deg is not None or decoded.tws_kts is not None or + decoded.aws_kts is not None): + self.wind_time = now + if decoded.twd_deg is not None: + self.twd_deg = decoded.twd_deg + if decoded.tws_kts is not None: + self.tws_kts = decoded.tws_kts + if decoded.awa_deg is not None: + self.awa_deg = decoded.awa_deg + if decoded.aws_kts is not None: + self.aws_kts = decoded.aws_kts + + # Update depth + if decoded.depth_m is not None: + self.depth_m = decoded.depth_m + self.depth_time = now + + # Update temperature + if decoded.water_temp_c is not None or decoded.air_temp_c is not None: + self.temp_time = now + if decoded.water_temp_c is not None: + self.water_temp_c = decoded.water_temp_c + if decoded.air_temp_c is not None: + self.air_temp_c = decoded.air_temp_c + + # Update pressure + if decoded.pressure_mbar is not None: + self.pressure_mbar = decoded.pressure_mbar + self.pressure_time = now + + # Update tanks + if decoded.tanks: + self.tanks.update(decoded.tanks) + self.tank_time = now + + # Update batteries + if decoded.batteries: + self.batteries.update(decoded.batteries) + self.battery_time = now + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary for JSON serialization. + + Returns: + Dictionary with all sensor data + """ + with self._lock: + return { + "timestamp": datetime.now().isoformat(), + "position": { + "latitude": self.latitude, + "longitude": self.longitude, + }, + "navigation": { + "heading_deg": round(self.heading_deg, 1) if self.heading_deg else None, + "cog_deg": round(self.cog_deg, 1) if self.cog_deg else None, + "sog_kts": round(self.sog_kts, 1) if self.sog_kts else None, + }, + "wind": { + "true_direction_deg": round(self.twd_deg, 1) if self.twd_deg else None, + "true_speed_kts": round(self.tws_kts, 1) if self.tws_kts else None, + "apparent_angle_deg": round(self.awa_deg, 1) if self.awa_deg else None, + "apparent_speed_kts": round(self.aws_kts, 1) if self.aws_kts else None, + }, + "depth": { + "meters": round(self.depth_m, 1) if self.depth_m else None, + "feet": round(self.depth_ft, 1) if self.depth_ft else None, + }, + "temperature": { + "water_c": round(self.water_temp_c, 1) if self.water_temp_c else None, + "water_f": round(self.water_temp_f, 1) if self.water_temp_f else None, + "air_c": round(self.air_temp_c, 1) if self.air_temp_c else None, + "air_f": round(self.air_temp_f, 1) if self.air_temp_f else None, + }, + "pressure": { + "mbar": round(self.pressure_mbar, 1) if self.pressure_mbar else None, + "inhg": round(self.pressure_inhg, 2) if self.pressure_inhg else None, + }, + "tanks": { + str(tank_id): { + "name": get_tank_name(tank_id), + "level_pct": round(level, 1), + "capacity_gal": get_tank_capacity(tank_id), + } for tank_id, level in self.tanks.items() + }, + "batteries": { + str(battery_id): { + "name": get_battery_name(battery_id), + "voltage_v": round(voltage, 2), + } for battery_id, voltage in self.batteries.items() + }, + "stats": { + "packets": self.packet_count, + "decoded": self.decode_count, + "uptime_s": round(self.uptime, 1), + } + } + + def get_age(self, data_type: str) -> Optional[float]: + """Get the age of a data type in seconds. + + Args: + data_type: One of 'gps', 'heading', 'wind', 'depth', 'temp', + 'tank', 'battery' + + Returns: + Age in seconds, or None if no data has been received + """ + time_map = { + 'gps': self.gps_time, + 'heading': self.heading_time, + 'wind': self.wind_time, + 'depth': self.depth_time, + 'temp': self.temp_time, + 'pressure': self.pressure_time, + 'tank': self.tank_time, + 'battery': self.battery_time, + } + with self._lock: + ts = time_map.get(data_type, 0) + if ts == 0: + return None + return time.time() - ts + + def is_stale(self, data_type: str, max_age: float = 10.0) -> bool: + """Check if a data type is stale. + + Args: + data_type: One of 'gps', 'heading', 'wind', 'depth', 'temp', + 'tank', 'battery' + max_age: Maximum age in seconds before data is considered stale + + Returns: + True if data is stale or missing + """ + age = self.get_age(data_type) + if age is None: + return True + return age > max_age + + def reset(self) -> None: + """Reset all data and statistics.""" + with self._lock: + self.latitude = None + self.longitude = None + self.heading_deg = None + self.cog_deg = None + self.sog_kts = None + self.twd_deg = None + self.tws_kts = None + self.awa_deg = None + self.aws_kts = None + self.depth_m = None + self.water_temp_c = None + self.air_temp_c = None + self.pressure_mbar = None + self.tanks.clear() + self.batteries.clear() + self.gps_time = 0 + self.heading_time = 0 + self.wind_time = 0 + self.depth_time = 0 + self.temp_time = 0 + self.pressure_time = 0 + self.tank_time = 0 + self.battery_time = 0 + self.packet_count = 0 + self.decode_count = 0 + self.start_time = time.time() diff --git a/axiom-nmea/raymarine_nmea/listeners/__init__.py b/axiom-nmea/raymarine_nmea/listeners/__init__.py new file mode 100644 index 0000000..9c9112e --- /dev/null +++ b/axiom-nmea/raymarine_nmea/listeners/__init__.py @@ -0,0 +1,11 @@ +""" +Data source listeners for Raymarine data. + +MulticastListener - Listen on UDP multicast groups for live data +PcapReader - Read packets from PCAP files for offline analysis +""" + +from .multicast import MulticastListener +from .pcap import PcapReader + +__all__ = ["MulticastListener", "PcapReader"] diff --git a/axiom-nmea/raymarine_nmea/listeners/multicast.py b/axiom-nmea/raymarine_nmea/listeners/multicast.py new file mode 100644 index 0000000..a66724f --- /dev/null +++ b/axiom-nmea/raymarine_nmea/listeners/multicast.py @@ -0,0 +1,184 @@ +""" +Multicast UDP listener for Raymarine data. + +Listens on multiple multicast groups simultaneously and decodes +incoming Raymarine packets. +""" + +import socket +import struct +import threading +from typing import List, Tuple, Optional, Callable + +from ..protocol.decoder import RaymarineDecoder, DecodedData +from ..data.store import SensorData +from ..sensors import MULTICAST_GROUPS + + +class MulticastListener: + """Listens on Raymarine multicast groups for sensor data. + + This class creates UDP sockets for each configured multicast group + and receives packets in separate threads. Decoded data is stored + in a thread-safe SensorData object. + + Example: + data = SensorData() + decoder = RaymarineDecoder() + listener = MulticastListener( + decoder=decoder, + sensor_data=data, + interface_ip="198.18.5.5" + ) + + listener.start() + try: + while True: + print(f"GPS: {data.latitude}, {data.longitude}") + time.sleep(1) + finally: + listener.stop() + """ + + def __init__( + self, + decoder: RaymarineDecoder, + sensor_data: SensorData, + interface_ip: str, + groups: Optional[List[Tuple[str, int]]] = None, + on_packet: Optional[Callable[[bytes, str, int], None]] = None, + on_decode: Optional[Callable[[DecodedData], None]] = None, + ): + """Initialize the multicast listener. + + Args: + decoder: RaymarineDecoder instance + sensor_data: SensorData instance to store decoded values + interface_ip: IP address of the network interface to use + groups: List of (multicast_group, port) tuples. + If None, uses default MULTICAST_GROUPS. + on_packet: Optional callback for each received packet. + Called with (packet_bytes, group, port). + on_decode: Optional callback for each decoded packet. + Called with DecodedData object. + """ + self.decoder = decoder + self.sensor_data = sensor_data + self.interface_ip = interface_ip + self.groups = groups or MULTICAST_GROUPS + self.on_packet = on_packet + self.on_decode = on_decode + + self.running = False + self.sockets: List[socket.socket] = [] + self.threads: List[threading.Thread] = [] + + def _create_socket(self, group: str, port: int) -> Optional[socket.socket]: + """Create a UDP socket for a multicast group. + + Args: + group: Multicast group address + port: UDP port number + + Returns: + Configured socket, or None on error + """ + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + # Enable port reuse if available + if hasattr(socket, 'SO_REUSEPORT'): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + + sock.bind(('', port)) + + # Join multicast group + mreq = struct.pack( + "4s4s", + socket.inet_aton(group), + socket.inet_aton(self.interface_ip) + ) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + + # Set timeout for graceful shutdown + sock.settimeout(1.0) + + return sock + + except Exception as e: + print(f"Error creating socket for {group}:{port}: {e}") + return None + + def _listen(self, sock: socket.socket, group: str, port: int) -> None: + """Listen on a socket and decode packets. + + Args: + sock: UDP socket + group: Multicast group (for logging) + port: Port number (for logging) + """ + while self.running: + try: + data, addr = sock.recvfrom(65535) + + # Optional packet callback + if self.on_packet: + self.on_packet(data, group, port) + + # Decode the packet + result = self.decoder.decode(data) + + # Update sensor data + self.sensor_data.update(result) + + # Optional decode callback + if self.on_decode and result.has_data(): + self.on_decode(result) + + except socket.timeout: + continue + except Exception as e: + if self.running: + print(f"Error on {group}:{port}: {e}") + + def start(self) -> None: + """Start listening on all multicast groups.""" + self.running = True + + for group, port in self.groups: + sock = self._create_socket(group, port) + if sock: + self.sockets.append(sock) + thread = threading.Thread( + target=self._listen, + args=(sock, group, port), + daemon=True, + name=f"listener-{group}:{port}" + ) + thread.start() + self.threads.append(thread) + print(f"Listening on {group}:{port}") + + def stop(self) -> None: + """Stop all listeners and close sockets.""" + self.running = False + + # Wait for threads to finish + for thread in self.threads: + thread.join(timeout=2.0) + + # Close sockets + for sock in self.sockets: + try: + sock.close() + except Exception: + pass + + self.threads.clear() + self.sockets.clear() + + @property + def is_running(self) -> bool: + """Check if the listener is running.""" + return self.running and any(t.is_alive() for t in self.threads) diff --git a/axiom-nmea/raymarine_nmea/listeners/pcap.py b/axiom-nmea/raymarine_nmea/listeners/pcap.py new file mode 100644 index 0000000..30740b9 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/listeners/pcap.py @@ -0,0 +1,210 @@ +""" +PCAP file reader for offline analysis. + +Reads Raymarine packets from PCAP capture files. +""" + +import struct +from typing import List, Iterator, Tuple, Optional + +from ..protocol.decoder import RaymarineDecoder, DecodedData +from ..data.store import SensorData + + +class PcapReader: + """Reads packets from PCAP capture files. + + Supports standard libpcap format (both little and big endian). + Extracts UDP payloads from Ethernet/IP/UDP frames. + + Example: + reader = PcapReader("capture.pcap") + decoder = RaymarineDecoder() + data = SensorData() + + for packet in reader: + result = decoder.decode(packet) + data.update(result) + + print(f"Processed {len(reader)} packets") + print(f"Final GPS: {data.latitude}, {data.longitude}") + """ + + # Ethernet header size + ETH_HEADER_SIZE = 14 + + # IP header minimum size + IP_HEADER_MIN_SIZE = 20 + + # UDP header size + UDP_HEADER_SIZE = 8 + + def __init__(self, filename: str): + """Initialize the PCAP reader. + + Args: + filename: Path to the PCAP file + """ + self.filename = filename + self._packets: Optional[List[bytes]] = None + + def _read_pcap(self) -> List[bytes]: + """Read and parse the PCAP file. + + Returns: + List of UDP payload bytes + """ + packets = [] + + with open(self.filename, 'rb') as f: + # Read global header + header = f.read(24) + if len(header) < 24: + return packets + + # Check magic number to determine endianness + magic = struct.unpack(' Optional[bytes]: + """Extract UDP payload from an Ethernet frame. + + Args: + frame: Raw Ethernet frame + + Returns: + UDP payload bytes, or None if not a UDP packet + """ + # Need at least Ethernet + IP + UDP headers + min_size = self.ETH_HEADER_SIZE + self.IP_HEADER_MIN_SIZE + self.UDP_HEADER_SIZE + if len(frame) < min_size: + return None + + # Check EtherType (offset 12-13) + ethertype = struct.unpack('>H', frame[12:14])[0] + if ethertype != 0x0800: # IPv4 + return None + + # Parse IP header + ip_start = self.ETH_HEADER_SIZE + ip_version_ihl = frame[ip_start] + ip_version = (ip_version_ihl >> 4) & 0x0F + ip_header_len = (ip_version_ihl & 0x0F) * 4 + + if ip_version != 4: + return None + + # Check protocol (offset 9 in IP header) + protocol = frame[ip_start + 9] + if protocol != 17: # UDP + return None + + # UDP payload starts after IP and UDP headers + udp_start = ip_start + ip_header_len + payload_start = udp_start + self.UDP_HEADER_SIZE + + if payload_start >= len(frame): + return None + + return frame[payload_start:] + + @property + def packets(self) -> List[bytes]: + """Get all packets from the PCAP file (cached).""" + if self._packets is None: + self._packets = self._read_pcap() + return self._packets + + def __len__(self) -> int: + """Return the number of packets.""" + return len(self.packets) + + def __iter__(self) -> Iterator[bytes]: + """Iterate over packets.""" + return iter(self.packets) + + def __getitem__(self, index: int) -> bytes: + """Get a packet by index.""" + return self.packets[index] + + def decode_all( + self, + decoder: Optional[RaymarineDecoder] = None + ) -> SensorData: + """Decode all packets and return aggregated sensor data. + + Args: + decoder: RaymarineDecoder instance, or None to create one + + Returns: + SensorData with all decoded values + """ + if decoder is None: + decoder = RaymarineDecoder() + + data = SensorData() + + for packet in self: + result = decoder.decode(packet) + data.update(result) + + return data + + def iter_decoded( + self, + decoder: Optional[RaymarineDecoder] = None + ) -> Iterator[Tuple[bytes, DecodedData]]: + """Iterate over packets and their decoded data. + + Args: + decoder: RaymarineDecoder instance, or None to create one + + Yields: + Tuples of (packet_bytes, DecodedData) + """ + if decoder is None: + decoder = RaymarineDecoder() + + for packet in self: + result = decoder.decode(packet) + yield packet, result + + @classmethod + def from_file(cls, filename: str) -> 'PcapReader': + """Create a PcapReader from a file path. + + Args: + filename: Path to PCAP file + + Returns: + PcapReader instance + """ + return cls(filename) diff --git a/axiom-nmea/raymarine_nmea/nmea/__init__.py b/axiom-nmea/raymarine_nmea/nmea/__init__.py new file mode 100644 index 0000000..c684b27 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/nmea/__init__.py @@ -0,0 +1,86 @@ +""" +NMEA 0183 sentence generation module. + +This module provides classes for generating standard NMEA 0183 sentences +from sensor data. Supported sentence types: + +GPS/Position: + - GGA: GPS Fix Data + - GLL: Geographic Position + - RMC: Recommended Minimum + +Navigation: + - HDG: Heading (magnetic with deviation/variation) + - HDT: Heading True + - VTG: Track Made Good and Ground Speed + - VHW: Water Speed and Heading + +Wind: + - MWV: Wind Speed and Angle + - MWD: Wind Direction and Speed + +Depth: + - DPT: Depth + - DBT: Depth Below Transducer + +Temperature: + - MTW: Water Temperature + - MTA: Air Temperature (proprietary extension) + +Transducer (tanks, batteries): + - XDR: Transducer Measurements +""" + +from .sentence import NMEASentence +from .generator import NMEAGenerator +from .server import NMEATcpServer + +# Import all sentence types +from .sentences import ( + # GPS + GGASentence, + GLLSentence, + RMCSentence, + # Navigation + HDGSentence, + HDTSentence, + VTGSentence, + VHWSentence, + # Wind + MWVSentence, + MWDSentence, + # Depth + DPTSentence, + DBTSentence, + # Temperature + MTWSentence, + MTASentence, + # Transducer + XDRSentence, +) + +__all__ = [ + "NMEASentence", + "NMEAGenerator", + "NMEATcpServer", + # GPS + "GGASentence", + "GLLSentence", + "RMCSentence", + # Navigation + "HDGSentence", + "HDTSentence", + "VTGSentence", + "VHWSentence", + # Wind + "MWVSentence", + "MWDSentence", + # Depth + "DPTSentence", + "DBTSentence", + # Temperature + "MTWSentence", + "MTASentence", + # Transducer + "XDRSentence", +] diff --git a/axiom-nmea/raymarine_nmea/nmea/generator.py b/axiom-nmea/raymarine_nmea/nmea/generator.py new file mode 100644 index 0000000..9e1d409 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/nmea/generator.py @@ -0,0 +1,523 @@ +""" +NMEA sentence generator. + +Generates complete sets of NMEA 0183 sentences from sensor data. +""" + +from datetime import datetime +from typing import List, Optional, Dict, Set + +from ..data.store import SensorData +from ..sensors import get_tank_name, get_battery_name + +from .sentences import ( + GGASentence, + GLLSentence, + RMCSentence, + HDGSentence, + HDTSentence, + VTGSentence, + VHWSentence, + MWVSentence, + MWDSentence, + DPTSentence, + DBTSentence, + MTWSentence, + MTASentence, + XDRSentence, +) + + +class NMEAGenerator: + """Generates NMEA 0183 sentences from sensor data. + + This class provides methods to generate various NMEA sentences + from SensorData. It can generate individual sentence types or + complete sets of all available sentences. + + Example: + generator = NMEAGenerator() + + # Generate all sentences + sentences = generator.generate_all(sensor_data) + for sentence in sentences: + print(sentence, end='') + + # Generate specific sentences + gga = generator.generate_gga(sensor_data) + mwd = generator.generate_mwd(sensor_data) + + Sentence Types Generated: + GPS: GGA, GLL, RMC + Navigation: HDG, HDT, VTG, VHW + Wind: MWV (apparent & true), MWD + Depth: DPT, DBT + Temperature: MTW, MTA + Transducers: XDR (tanks, batteries) + """ + + # Default magnetic variation (can be overridden) + DEFAULT_MAG_VARIATION = 0.0 + + # Sentence types that can be enabled/disabled + SENTENCE_TYPES = { + 'gga', 'gll', 'rmc', # GPS + 'hdg', 'hdt', 'vtg', 'vhw', # Navigation + 'mwv_apparent', 'mwv_true', 'mwd', # Wind + 'dpt', 'dbt', # Depth + 'mtw', 'mta', # Temperature + 'xdr_tanks', 'xdr_batteries', 'xdr_pressure', # Transducers + } + + def __init__( + self, + mag_variation: Optional[float] = None, + enabled_sentences: Optional[Set[str]] = None, + transducer_offset: float = 0.0, + ): + """Initialize the NMEA generator. + + Args: + mag_variation: Magnetic variation in degrees (positive=East) + enabled_sentences: Set of sentence types to generate. + If None, all sentences are enabled. + transducer_offset: Depth transducer offset in meters + (positive = to waterline, negative = to keel) + """ + self.mag_variation = mag_variation or self.DEFAULT_MAG_VARIATION + self.enabled = enabled_sentences or self.SENTENCE_TYPES.copy() + self.transducer_offset = transducer_offset + + def is_enabled(self, sentence_type: str) -> bool: + """Check if a sentence type is enabled.""" + return sentence_type in self.enabled + + def enable(self, sentence_type: str) -> None: + """Enable a sentence type.""" + if sentence_type in self.SENTENCE_TYPES: + self.enabled.add(sentence_type) + + def disable(self, sentence_type: str) -> None: + """Disable a sentence type.""" + self.enabled.discard(sentence_type) + + def generate_all(self, data: SensorData) -> List[str]: + """Generate all enabled NMEA sentences. + + Args: + data: SensorData object with current sensor values + + Returns: + List of NMEA sentence strings (with CRLF) + """ + sentences = [] + now = datetime.utcnow() + + # GPS sentences + if self.is_enabled('gga'): + s = self.generate_gga(data, now) + if s: + sentences.append(s) + + if self.is_enabled('gll'): + s = self.generate_gll(data, now) + if s: + sentences.append(s) + + if self.is_enabled('rmc'): + s = self.generate_rmc(data, now) + if s: + sentences.append(s) + + # Navigation sentences + if self.is_enabled('hdg'): + s = self.generate_hdg(data) + if s: + sentences.append(s) + + if self.is_enabled('hdt'): + s = self.generate_hdt(data) + if s: + sentences.append(s) + + if self.is_enabled('vtg'): + s = self.generate_vtg(data) + if s: + sentences.append(s) + + if self.is_enabled('vhw'): + s = self.generate_vhw(data) + if s: + sentences.append(s) + + # Wind sentences + if self.is_enabled('mwv_apparent'): + s = self.generate_mwv_apparent(data) + if s: + sentences.append(s) + + if self.is_enabled('mwv_true'): + s = self.generate_mwv_true(data) + if s: + sentences.append(s) + + if self.is_enabled('mwd'): + s = self.generate_mwd(data) + if s: + sentences.append(s) + + # Depth sentences + if self.is_enabled('dpt'): + s = self.generate_dpt(data) + if s: + sentences.append(s) + + if self.is_enabled('dbt'): + s = self.generate_dbt(data) + if s: + sentences.append(s) + + # Temperature sentences + if self.is_enabled('mtw'): + s = self.generate_mtw(data) + if s: + sentences.append(s) + + if self.is_enabled('mta'): + s = self.generate_mta(data) + if s: + sentences.append(s) + + # Transducer sentences + if self.is_enabled('xdr_tanks'): + s = self.generate_xdr_tanks(data) + if s: + sentences.append(s) + + if self.is_enabled('xdr_batteries'): + s = self.generate_xdr_batteries(data) + if s: + sentences.append(s) + + if self.is_enabled('xdr_pressure'): + s = self.generate_xdr_pressure(data) + if s: + sentences.append(s) + + return sentences + + # GPS Sentences + + def generate_gga( + self, + data: SensorData, + time: Optional[datetime] = None + ) -> Optional[str]: + """Generate GGA (GPS Fix Data) sentence.""" + with data._lock: + lat = data.latitude + lon = data.longitude + + if lat is None or lon is None: + return None + + sentence = GGASentence( + latitude=lat, + longitude=lon, + time=time, + ) + return sentence.to_nmea() + + def generate_gll( + self, + data: SensorData, + time: Optional[datetime] = None + ) -> Optional[str]: + """Generate GLL (Geographic Position) sentence.""" + with data._lock: + lat = data.latitude + lon = data.longitude + + if lat is None or lon is None: + return None + + sentence = GLLSentence( + latitude=lat, + longitude=lon, + time=time, + ) + return sentence.to_nmea() + + def generate_rmc( + self, + data: SensorData, + time: Optional[datetime] = None + ) -> Optional[str]: + """Generate RMC (Recommended Minimum) sentence.""" + with data._lock: + lat = data.latitude + lon = data.longitude + sog = data.sog_kts + cog = data.cog_deg + + if lat is None or lon is None: + return None + + sentence = RMCSentence( + latitude=lat, + longitude=lon, + time=time, + sog=sog, + cog=cog, + mag_var=self.mag_variation, + ) + return sentence.to_nmea() + + # Navigation Sentences + + def generate_hdg(self, data: SensorData) -> Optional[str]: + """Generate HDG (Heading, Deviation & Variation) sentence.""" + with data._lock: + heading = data.heading_deg + + if heading is None: + return None + + # Convert true heading to magnetic + heading_mag = (heading - self.mag_variation) % 360 + + sentence = HDGSentence( + heading=heading_mag, + deviation=0.0, + variation=self.mag_variation, + ) + return sentence.to_nmea() + + def generate_hdt(self, data: SensorData) -> Optional[str]: + """Generate HDT (Heading True) sentence.""" + with data._lock: + heading = data.heading_deg + + if heading is None: + return None + + sentence = HDTSentence(heading=heading) + return sentence.to_nmea() + + def generate_vtg(self, data: SensorData) -> Optional[str]: + """Generate VTG (Track Made Good) sentence.""" + with data._lock: + cog = data.cog_deg + sog = data.sog_kts + + # Need at least one value + if cog is None and sog is None: + return None + + cog_mag = None + if cog is not None: + cog_mag = (cog - self.mag_variation) % 360 + + sentence = VTGSentence( + cog_true=cog, + cog_mag=cog_mag, + sog_kts=sog, + ) + return sentence.to_nmea() + + def generate_vhw(self, data: SensorData) -> Optional[str]: + """Generate VHW (Water Speed and Heading) sentence.""" + with data._lock: + heading = data.heading_deg + # Note: We don't have water speed, so just use heading + + if heading is None: + return None + + heading_mag = (heading - self.mag_variation) % 360 + + sentence = VHWSentence( + heading_true=heading, + heading_mag=heading_mag, + speed_kts=None, # No water speed available + ) + return sentence.to_nmea() + + # Wind Sentences + + def generate_mwv_apparent(self, data: SensorData) -> Optional[str]: + """Generate MWV sentence for apparent wind.""" + with data._lock: + awa = data.awa_deg + aws = data.aws_kts + + if awa is None and aws is None: + return None + + sentence = MWVSentence( + angle=awa, + reference="R", # Relative/Apparent + speed=aws, + speed_units="N", # Knots + ) + return sentence.to_nmea() + + def generate_mwv_true(self, data: SensorData) -> Optional[str]: + """Generate MWV sentence for true wind.""" + with data._lock: + twd = data.twd_deg + tws = data.tws_kts + heading = data.heading_deg + + if twd is None and tws is None: + return None + + # Calculate true wind angle relative to bow + twa = None + if twd is not None and heading is not None: + twa = (twd - heading) % 360 + if twa > 180: + twa = twa - 360 # Normalize to -180 to 180 + + sentence = MWVSentence( + angle=twa if twa is not None else twd, + reference="T", # True + speed=tws, + speed_units="N", # Knots + ) + return sentence.to_nmea() + + def generate_mwd(self, data: SensorData) -> Optional[str]: + """Generate MWD (Wind Direction and Speed) sentence.""" + with data._lock: + twd = data.twd_deg + tws = data.tws_kts + + if twd is None and tws is None: + return None + + # Calculate magnetic wind direction + twd_mag = None + if twd is not None: + twd_mag = (twd - self.mag_variation) % 360 + + sentence = MWDSentence( + direction_true=twd, + direction_mag=twd_mag, + speed_kts=tws, + ) + return sentence.to_nmea() + + # Depth Sentences + + def generate_dpt(self, data: SensorData) -> Optional[str]: + """Generate DPT (Depth) sentence.""" + with data._lock: + depth = data.depth_m + + if depth is None: + return None + + sentence = DPTSentence( + depth_m=depth, + offset_m=self.transducer_offset, + ) + return sentence.to_nmea() + + def generate_dbt(self, data: SensorData) -> Optional[str]: + """Generate DBT (Depth Below Transducer) sentence.""" + with data._lock: + depth = data.depth_m + + if depth is None: + return None + + sentence = DBTSentence(depth_m=depth) + return sentence.to_nmea() + + # Temperature Sentences + + def generate_mtw(self, data: SensorData) -> Optional[str]: + """Generate MTW (Water Temperature) sentence.""" + with data._lock: + temp = data.water_temp_c + + if temp is None: + return None + + sentence = MTWSentence(temp_c=temp) + return sentence.to_nmea() + + def generate_mta(self, data: SensorData) -> Optional[str]: + """Generate MTA (Air Temperature) sentence.""" + with data._lock: + temp = data.air_temp_c + + if temp is None: + return None + + sentence = MTASentence(temp_c=temp) + return sentence.to_nmea() + + # Transducer Sentences + + def generate_xdr_tanks(self, data: SensorData) -> Optional[str]: + """Generate XDR sentence for tank levels.""" + with data._lock: + tanks = dict(data.tanks) + + if not tanks: + return None + + # Build name mapping + names = {tid: get_tank_name(tid) for tid in tanks} + + sentence = XDRSentence.for_tanks(tanks, names) + return sentence.to_nmea() + + def generate_xdr_batteries(self, data: SensorData) -> Optional[str]: + """Generate XDR sentence for battery voltages.""" + with data._lock: + batteries = dict(data.batteries) + + if not batteries: + return None + + # Build name mapping + names = {bid: get_battery_name(bid) for bid in batteries} + + sentence = XDRSentence.for_batteries(batteries, names) + return sentence.to_nmea() + + def generate_xdr_pressure(self, data: SensorData) -> Optional[str]: + """Generate XDR sentence for barometric pressure.""" + with data._lock: + pressure = data.pressure_mbar + + if pressure is None: + return None + + sentence = XDRSentence.for_pressure(pressure) + return sentence.to_nmea() + + def generate_xdr_all(self, data: SensorData) -> List[str]: + """Generate all XDR sentences (tanks, batteries, pressure). + + Returns separate sentences for each type to avoid + exceeding NMEA sentence length limits. + """ + sentences = [] + + tanks_xdr = self.generate_xdr_tanks(data) + if tanks_xdr: + sentences.append(tanks_xdr) + + batteries_xdr = self.generate_xdr_batteries(data) + if batteries_xdr: + sentences.append(batteries_xdr) + + pressure_xdr = self.generate_xdr_pressure(data) + if pressure_xdr: + sentences.append(pressure_xdr) + + return sentences diff --git a/axiom-nmea/raymarine_nmea/nmea/sentence.py b/axiom-nmea/raymarine_nmea/nmea/sentence.py new file mode 100644 index 0000000..1268a8b --- /dev/null +++ b/axiom-nmea/raymarine_nmea/nmea/sentence.py @@ -0,0 +1,152 @@ +""" +Base class for NMEA 0183 sentences. + +NMEA 0183 Sentence Format: + $XXYYY,field1,field2,...,fieldN*CC + +Where: + $ = Start delimiter + XX = Talker ID (e.g., GP, II, WI) + YYY = Sentence type (e.g., GGA, RMC) + , = Field delimiter + * = Checksum delimiter + CC = Two-digit hex checksum (XOR of all chars between $ and *) + = Carriage return and line feed +""" + +from abc import ABC, abstractmethod +from datetime import datetime +from typing import Optional + + +class NMEASentence(ABC): + """Abstract base class for NMEA 0183 sentences. + + Subclasses must implement: + - sentence_type: The 3-character sentence type (e.g., "GGA") + - format_fields(): Returns the comma-separated field data + + Example: + class GGASentence(NMEASentence): + sentence_type = "GGA" + + def format_fields(self) -> str: + return "123456.00,2456.123,N,08037.456,W,1,08,0.9,..." + """ + + # Talker ID for this sentence (default: II for integrated instrumentation) + talker_id: str = "II" + + # Sentence type (e.g., "GGA", "RMC") + sentence_type: str = "" + + @abstractmethod + def format_fields(self) -> Optional[str]: + """Format the sentence fields. + + Returns: + Comma-separated field string, or None if sentence cannot be generated + """ + pass + + @staticmethod + def calculate_checksum(sentence: str) -> str: + """Calculate NMEA checksum. + + The checksum is the XOR of all characters between $ and *. + + Args: + sentence: The sentence content (without $ prefix and * suffix) + + Returns: + Two-character hex checksum + """ + checksum = 0 + for char in sentence: + checksum ^= ord(char) + return f"{checksum:02X}" + + def to_nmea(self) -> Optional[str]: + """Generate the complete NMEA sentence. + + Returns: + Complete NMEA sentence with $ prefix, checksum, and CRLF, + or None if sentence cannot be generated + """ + fields = self.format_fields() + if fields is None: + return None + + # Build sentence content (between $ and *) + content = f"{self.talker_id}{self.sentence_type},{fields}" + + # Calculate checksum + checksum = self.calculate_checksum(content) + + # Return complete sentence + return f"${content}*{checksum}\r\n" + + def __str__(self) -> str: + """Return the NMEA sentence as a string.""" + result = self.to_nmea() + return result if result else "" + + @staticmethod + def format_latitude(lat: float) -> str: + """Format latitude for NMEA (DDMM.MMMMM,N/S). + + Args: + lat: Latitude in decimal degrees (-90 to 90) + + Returns: + Formatted string like "2456.12345,N" + """ + hemisphere = 'N' if lat >= 0 else 'S' + lat = abs(lat) + degrees = int(lat) + minutes = (lat - degrees) * 60 + return f"{degrees:02d}{minutes:09.6f},{hemisphere}" + + @staticmethod + def format_longitude(lon: float) -> str: + """Format longitude for NMEA (DDDMM.MMMMM,E/W). + + Args: + lon: Longitude in decimal degrees (-180 to 180) + + Returns: + Formatted string like "08037.45678,W" + """ + hemisphere = 'E' if lon >= 0 else 'W' + lon = abs(lon) + degrees = int(lon) + minutes = (lon - degrees) * 60 + return f"{degrees:03d}{minutes:09.6f},{hemisphere}" + + @staticmethod + def format_time(dt: Optional[datetime] = None) -> str: + """Format time for NMEA (HHMMSS.SS). + + Args: + dt: Datetime object, or None for current time + + Returns: + Formatted string like "123456.00" + """ + if dt is None: + dt = datetime.utcnow() + return dt.strftime("%H%M%S.00") + + @staticmethod + def format_date(dt: Optional[datetime] = None) -> str: + """Format date for NMEA (DDMMYY). + + Args: + dt: Datetime object, or None for current time + + Returns: + Formatted string like "231224" + """ + if dt is None: + dt = datetime.utcnow() + return dt.strftime("%d%m%y") diff --git a/axiom-nmea/raymarine_nmea/nmea/sentences/__init__.py b/axiom-nmea/raymarine_nmea/nmea/sentences/__init__.py new file mode 100644 index 0000000..ba786a1 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/nmea/sentences/__init__.py @@ -0,0 +1,33 @@ +""" +NMEA sentence implementations. +""" + +from .gps import GGASentence, GLLSentence, RMCSentence +from .navigation import HDGSentence, HDTSentence, VTGSentence, VHWSentence +from .wind import MWVSentence, MWDSentence +from .depth import DPTSentence, DBTSentence +from .temperature import MTWSentence, MTASentence +from .transducer import XDRSentence + +__all__ = [ + # GPS + "GGASentence", + "GLLSentence", + "RMCSentence", + # Navigation + "HDGSentence", + "HDTSentence", + "VTGSentence", + "VHWSentence", + # Wind + "MWVSentence", + "MWDSentence", + # Depth + "DPTSentence", + "DBTSentence", + # Temperature + "MTWSentence", + "MTASentence", + # Transducer + "XDRSentence", +] diff --git a/axiom-nmea/raymarine_nmea/nmea/sentences/depth.py b/axiom-nmea/raymarine_nmea/nmea/sentences/depth.py new file mode 100644 index 0000000..6ecd0b7 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/nmea/sentences/depth.py @@ -0,0 +1,113 @@ +""" +Depth-related NMEA sentences. + +DPT - Depth +DBT - Depth Below Transducer +""" + +from typing import Optional + +from ..sentence import NMEASentence + + +class DPTSentence(NMEASentence): + """DPT - Depth of Water. + + Format: + $IIDPT,D.D,O.O,R.R*CC + + Fields: + 1. Depth in meters (relative to transducer) + 2. Offset from transducer in meters + - Positive = distance from transducer to water line + - Negative = distance from transducer to keel + 3. Maximum range scale in use (optional) + + Example: + $IIDPT,12.5,0.5,100*4C + + Note: + To get depth below keel: depth + offset (when offset is negative) + To get depth below surface: depth + offset (when offset is positive) + """ + + talker_id = "II" + sentence_type = "DPT" + + def __init__( + self, + depth_m: Optional[float] = None, + offset_m: float = 0.0, + max_range: Optional[float] = None, + ): + """Initialize DPT sentence. + + Args: + depth_m: Depth in meters (relative to transducer) + offset_m: Offset from transducer in meters + max_range: Maximum range scale in meters + """ + self.depth_m = depth_m + self.offset_m = offset_m + self.max_range = max_range + + def format_fields(self) -> Optional[str]: + """Format DPT fields.""" + if self.depth_m is None: + return None + + range_str = f"{self.max_range:.0f}" if self.max_range is not None else "" + + return f"{self.depth_m:.1f},{self.offset_m:.1f},{range_str}" + + +class DBTSentence(NMEASentence): + """DBT - Depth Below Transducer. + + Format: + $IIDBT,D.D,f,D.D,M,D.D,F*CC + + Fields: + 1. Depth in feet + 2. f = Feet + 3. Depth in meters + 4. M = Meters + 5. Depth in fathoms + 6. F = Fathoms + + Example: + $IIDBT,41.0,f,12.5,M,6.8,F*2B + + Note: + Depth is measured from the transducer to the bottom. + Does not include offset to keel or waterline. + """ + + talker_id = "II" + sentence_type = "DBT" + + # Conversion constants + FEET_PER_METER = 3.28084 + FATHOMS_PER_METER = 0.546807 + + def __init__(self, depth_m: Optional[float] = None): + """Initialize DBT sentence. + + Args: + depth_m: Depth in meters + """ + self.depth_m = depth_m + + def format_fields(self) -> Optional[str]: + """Format DBT fields.""" + if self.depth_m is None: + return None + + depth_ft = self.depth_m * self.FEET_PER_METER + depth_fathoms = self.depth_m * self.FATHOMS_PER_METER + + return ( + f"{depth_ft:.1f},f," + f"{self.depth_m:.1f},M," + f"{depth_fathoms:.1f},F" + ) diff --git a/axiom-nmea/raymarine_nmea/nmea/sentences/gps.py b/axiom-nmea/raymarine_nmea/nmea/sentences/gps.py new file mode 100644 index 0000000..85a5b11 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/nmea/sentences/gps.py @@ -0,0 +1,266 @@ +""" +GPS-related NMEA sentences. + +GGA - GPS Fix Data +GLL - Geographic Position +RMC - Recommended Minimum Navigation Information +""" + +from datetime import datetime +from typing import Optional + +from ..sentence import NMEASentence + + +class GGASentence(NMEASentence): + """GGA - GPS Fix Data. + + Format: + $GPGGA,HHMMSS.SS,DDMM.MMMMM,N,DDDMM.MMMMM,W,Q,SS,H.H,A.A,M,G.G,M,A.A,XXXX*CC + + Fields: + 1. Time (UTC) - HHMMSS.SS + 2. Latitude - DDMM.MMMMM + 3. N/S indicator + 4. Longitude - DDDMM.MMMMM + 5. E/W indicator + 6. GPS Quality (0=invalid, 1=GPS, 2=DGPS, 4=RTK fixed, 5=RTK float) + 7. Number of satellites + 8. HDOP (Horizontal Dilution of Precision) + 9. Altitude above mean sea level + 10. Altitude units (M) + 11. Geoidal separation + 12. Geoidal separation units (M) + 13. Age of differential GPS data + 14. Differential reference station ID + + Example: + $GPGGA,123519.00,4807.038000,N,01131.000000,E,1,08,0.9,545.4,M,47.0,M,,*47 + """ + + talker_id = "GP" + sentence_type = "GGA" + + def __init__( + self, + latitude: Optional[float] = None, + longitude: Optional[float] = None, + time: Optional[datetime] = None, + quality: int = 1, + num_satellites: int = 8, + hdop: float = 1.0, + altitude: Optional[float] = None, + geoid_sep: Optional[float] = None, + ): + """Initialize GGA sentence. + + Args: + latitude: Latitude in decimal degrees + longitude: Longitude in decimal degrees + time: UTC time, or None for current time + quality: GPS quality indicator (1=GPS, 2=DGPS) + num_satellites: Number of satellites in use + hdop: Horizontal dilution of precision + altitude: Altitude above MSL in meters + geoid_sep: Geoidal separation in meters + """ + self.latitude = latitude + self.longitude = longitude + self.time = time + self.quality = quality + self.num_satellites = num_satellites + self.hdop = hdop + self.altitude = altitude + self.geoid_sep = geoid_sep + + def format_fields(self) -> Optional[str]: + """Format GGA fields.""" + if self.latitude is None or self.longitude is None: + return None + + time_str = self.format_time(self.time) + lat_str = self.format_latitude(self.latitude) + lon_str = self.format_longitude(self.longitude) + + # Format altitude - not yet available from Raymarine + if self.altitude is not None: + alt_str = f"{self.altitude:.1f},M" + else: + alt_str = "0.0,M" # Default to 0.0 for parsers that require a value + + # Format geoidal separation - not available from Raymarine + if self.geoid_sep is not None: + geoid_str = f"{self.geoid_sep:.1f},M" + else: + geoid_str = "-22.0,M" # Default typical value for compatibility + + return ( + f"{time_str}," + f"{lat_str}," + f"{lon_str}," + f"{self.quality}," + f"{self.num_satellites:02d}," + f"{self.hdop:.1f}," + f"{alt_str}," + f"{geoid_str}," + f"," # Age of DGPS + ) # DGPS station ID (empty) + + +class GLLSentence(NMEASentence): + """GLL - Geographic Position (Latitude/Longitude). + + Format: + $GPGLL,DDMM.MMMMM,N,DDDMM.MMMMM,W,HHMMSS.SS,A,A*CC + + Fields: + 1. Latitude - DDMM.MMMMM + 2. N/S indicator + 3. Longitude - DDDMM.MMMMM + 4. E/W indicator + 5. Time (UTC) - HHMMSS.SS + 6. Status (A=valid, V=invalid) + 7. Mode indicator (A=autonomous, D=differential, N=invalid) + + Example: + $GPGLL,4916.45000,N,12311.12000,W,225444.00,A,A*6A + """ + + talker_id = "GP" + sentence_type = "GLL" + + def __init__( + self, + latitude: Optional[float] = None, + longitude: Optional[float] = None, + time: Optional[datetime] = None, + status: str = "A", + mode: str = "A", + ): + """Initialize GLL sentence. + + Args: + latitude: Latitude in decimal degrees + longitude: Longitude in decimal degrees + time: UTC time, or None for current time + status: Status (A=valid, V=invalid) + mode: Mode indicator (A=autonomous, D=differential) + """ + self.latitude = latitude + self.longitude = longitude + self.time = time + self.status = status + self.mode = mode + + def format_fields(self) -> Optional[str]: + """Format GLL fields.""" + if self.latitude is None or self.longitude is None: + return None + + lat_str = self.format_latitude(self.latitude) + lon_str = self.format_longitude(self.longitude) + time_str = self.format_time(self.time) + + return f"{lat_str},{lon_str},{time_str},{self.status},{self.mode}" + + +class RMCSentence(NMEASentence): + """RMC - Recommended Minimum Navigation Information. + + Format: + $GPRMC,HHMMSS.SS,A,DDMM.MMMMM,N,DDDMM.MMMMM,W,S.S,C.C,DDMMYY,M.M,E,A*CC + + Fields: + 1. Time (UTC) - HHMMSS.SS + 2. Status (A=valid, V=warning) + 3. Latitude - DDMM.MMMMM + 4. N/S indicator + 5. Longitude - DDDMM.MMMMM + 6. E/W indicator + 7. Speed over ground (knots) + 8. Course over ground (degrees true) + 9. Date - DDMMYY + 10. Magnetic variation (degrees) + 11. Magnetic variation direction (E/W) + 12. Mode indicator (A=autonomous, D=differential) + + Example: + $GPRMC,225446.00,A,4916.45000,N,12311.12000,W,000.5,054.7,191194,020.3,E,A*68 + """ + + talker_id = "GP" + sentence_type = "RMC" + + def __init__( + self, + latitude: Optional[float] = None, + longitude: Optional[float] = None, + time: Optional[datetime] = None, + status: str = "A", + sog: Optional[float] = None, + cog: Optional[float] = None, + mag_var: Optional[float] = None, + mode: str = "A", + ): + """Initialize RMC sentence. + + Args: + latitude: Latitude in decimal degrees + longitude: Longitude in decimal degrees + time: UTC time, or None for current time + status: Status (A=valid, V=warning) + sog: Speed over ground in knots + cog: Course over ground in degrees true + mag_var: Magnetic variation in degrees (positive=E, negative=W) + mode: Mode indicator (A=autonomous, D=differential) + """ + self.latitude = latitude + self.longitude = longitude + self.time = time + self.status = status + self.sog = sog + self.cog = cog + self.mag_var = mag_var + self.mode = mode + + def format_fields(self) -> Optional[str]: + """Format RMC fields.""" + if self.latitude is None or self.longitude is None: + return None + + dt = self.time if self.time else datetime.utcnow() + time_str = self.format_time(dt) + date_str = self.format_date(dt) + lat_str = self.format_latitude(self.latitude) + lon_str = self.format_longitude(self.longitude) + + # Format SOG - sourced from Field 5.3 + if self.sog is not None: + sog_str = f"{self.sog:.1f}" + else: + sog_str = "0.0" # Default to 0.0 for parsers that require a value + + # Format COG - sourced from Field 5.1 + if self.cog is not None: + cog_str = f"{self.cog:.1f}" + else: + cog_str = "0.0" # Default to 0.0 for parsers that require a value + + # Format magnetic variation + if self.mag_var is not None: + mag_dir = 'E' if self.mag_var >= 0 else 'W' + mag_str = f"{abs(self.mag_var):.1f},{mag_dir}" + else: + mag_str = "," + + return ( + f"{time_str}," + f"{self.status}," + f"{lat_str}," + f"{lon_str}," + f"{sog_str}," + f"{cog_str}," + f"{date_str}," + f"{mag_str}," + f"{self.mode}" + ) diff --git a/axiom-nmea/raymarine_nmea/nmea/sentences/navigation.py b/axiom-nmea/raymarine_nmea/nmea/sentences/navigation.py new file mode 100644 index 0000000..7ab1682 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/nmea/sentences/navigation.py @@ -0,0 +1,252 @@ +""" +Navigation-related NMEA sentences. + +HDG - Heading (Deviation & Variation) +HDT - Heading True +VTG - Track Made Good and Ground Speed +VHW - Water Speed and Heading +""" + +from typing import Optional + +from ..sentence import NMEASentence + + +class HDGSentence(NMEASentence): + """HDG - Heading, Deviation & Variation. + + Format: + $IIHDG,H.H,D.D,E,V.V,E*CC + + Fields: + 1. Heading (magnetic) in degrees + 2. Magnetic deviation in degrees + 3. Deviation direction (E/W) + 4. Magnetic variation in degrees + 5. Variation direction (E/W) + + Example: + $IIHDG,238.5,0.0,E,12.6,W*5F + """ + + talker_id = "II" + sentence_type = "HDG" + + def __init__( + self, + heading: Optional[float] = None, + deviation: Optional[float] = None, + variation: Optional[float] = None, + ): + """Initialize HDG sentence. + + Args: + heading: Magnetic heading in degrees (0-360) + deviation: Magnetic deviation in degrees (positive=E, negative=W) + variation: Magnetic variation in degrees (positive=E, negative=W) + """ + self.heading = heading + self.deviation = deviation + self.variation = variation + + def format_fields(self) -> Optional[str]: + """Format HDG fields.""" + if self.heading is None: + return None + + heading_str = f"{self.heading:.1f}" + + # Format deviation - not yet available from Raymarine + if self.deviation is not None: + dev_dir = 'E' if self.deviation >= 0 else 'W' + dev_str = f"{abs(self.deviation):.1f},{dev_dir}" + else: + dev_str = "0.0,E" # Default to 0.0 for parsers that require a value + + # Format variation - configured value, not from Raymarine + if self.variation is not None: + var_dir = 'E' if self.variation >= 0 else 'W' + var_str = f"{abs(self.variation):.1f},{var_dir}" + else: + var_str = "0.0,E" # Default to 0.0 for parsers that require a value + + return f"{heading_str},{dev_str},{var_str}" + + +class HDTSentence(NMEASentence): + """HDT - Heading True. + + Format: + $IIHDT,H.H,T*CC + + Fields: + 1. Heading (true) in degrees + 2. T = True + + Example: + $IIHDT,238.5,T*1C + """ + + talker_id = "II" + sentence_type = "HDT" + + def __init__(self, heading: Optional[float] = None): + """Initialize HDT sentence. + + Args: + heading: True heading in degrees (0-360) + """ + self.heading = heading + + def format_fields(self) -> Optional[str]: + """Format HDT fields.""" + if self.heading is None: + return None + return f"{self.heading:.1f},T" + + +class VTGSentence(NMEASentence): + """VTG - Track Made Good and Ground Speed. + + Format: + $IIVTG,C.C,T,C.C,M,S.S,N,S.S,K,M*CC + + Fields: + 1. Course over ground (true) in degrees + 2. T = True + 3. Course over ground (magnetic) in degrees + 4. M = Magnetic + 5. Speed over ground in knots + 6. N = Knots + 7. Speed over ground in km/h + 8. K = Km/h + 9. Mode indicator (A=autonomous, D=differential) + + Example: + $IIVTG,054.7,T,034.4,M,005.5,N,010.2,K,A*28 + """ + + talker_id = "II" + sentence_type = "VTG" + + def __init__( + self, + cog_true: Optional[float] = None, + cog_mag: Optional[float] = None, + sog_kts: Optional[float] = None, + mode: str = "A", + ): + """Initialize VTG sentence. + + Args: + cog_true: Course over ground (true) in degrees + cog_mag: Course over ground (magnetic) in degrees + sog_kts: Speed over ground in knots + mode: Mode indicator (A=autonomous, D=differential) + """ + self.cog_true = cog_true + self.cog_mag = cog_mag + self.sog_kts = sog_kts + self.mode = mode + + def format_fields(self) -> Optional[str]: + """Format VTG fields.""" + # Format COG true - sourced from Field 5.1 + if self.cog_true is not None: + cog_t_str = f"{self.cog_true:.1f}" + else: + cog_t_str = "0.0" # Default to 0.0 for parsers that require a value + + # Format COG magnetic - derived from true COG + if self.cog_mag is not None: + cog_m_str = f"{self.cog_mag:.1f}" + else: + cog_m_str = "0.0" # Default to 0.0 for parsers that require a value + + # Format SOG - sourced from Field 5.3 + if self.sog_kts is not None: + sog_kts_str = f"{self.sog_kts:.1f}" + sog_kmh = self.sog_kts * 1.852 + sog_kmh_str = f"{sog_kmh:.1f}" + else: + sog_kts_str = "0.0" # Default to 0.0 for parsers that require a value + sog_kmh_str = "0.0" + + return ( + f"{cog_t_str},T," + f"{cog_m_str},M," + f"{sog_kts_str},N," + f"{sog_kmh_str},K," + f"{self.mode}" + ) + + +class VHWSentence(NMEASentence): + """VHW - Water Speed and Heading. + + Format: + $IIVHW,H.H,T,H.H,M,S.S,N,S.S,K*CC + + Fields: + 1. Heading (true) in degrees + 2. T = True + 3. Heading (magnetic) in degrees + 4. M = Magnetic + 5. Speed (water) in knots + 6. N = Knots + 7. Speed (water) in km/h + 8. K = Km/h + + Example: + $IIVHW,238.5,T,225.9,M,4.5,N,8.3,K*5D + """ + + talker_id = "II" + sentence_type = "VHW" + + def __init__( + self, + heading_true: Optional[float] = None, + heading_mag: Optional[float] = None, + speed_kts: Optional[float] = None, + ): + """Initialize VHW sentence. + + Args: + heading_true: Heading (true) in degrees + heading_mag: Heading (magnetic) in degrees + speed_kts: Speed through water in knots + """ + self.heading_true = heading_true + self.heading_mag = heading_mag + self.speed_kts = speed_kts + + def format_fields(self) -> Optional[str]: + """Format VHW fields.""" + # Format headings - true heading sourced from Field 3.2 + if self.heading_true is not None: + hdg_t_str = f"{self.heading_true:.1f}" + else: + hdg_t_str = "0.0" # Default to 0.0 for parsers that require a value + + # Magnetic heading derived from true heading + if self.heading_mag is not None: + hdg_m_str = f"{self.heading_mag:.1f}" + else: + hdg_m_str = "0.0" # Default to 0.0 for parsers that require a value + + # Format speed - water speed not yet available from Raymarine + if self.speed_kts is not None: + spd_kts_str = f"{self.speed_kts:.1f}" + spd_kmh = self.speed_kts * 1.852 + spd_kmh_str = f"{spd_kmh:.1f}" + else: + spd_kts_str = "0.0" # Default to 0.0 for parsers that require a value + spd_kmh_str = "0.0" + + return ( + f"{hdg_t_str},T," + f"{hdg_m_str},M," + f"{spd_kts_str},N," + f"{spd_kmh_str},K" + ) diff --git a/axiom-nmea/raymarine_nmea/nmea/sentences/temperature.py b/axiom-nmea/raymarine_nmea/nmea/sentences/temperature.py new file mode 100644 index 0000000..3041049 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/nmea/sentences/temperature.py @@ -0,0 +1,82 @@ +""" +Temperature-related NMEA sentences. + +MTW - Water Temperature +MTA - Air Temperature (proprietary extension) +""" + +from typing import Optional + +from ..sentence import NMEASentence + + +class MTWSentence(NMEASentence): + """MTW - Water Temperature. + + Format: + $IIMTW,T.T,C*CC + + Fields: + 1. Water temperature + 2. Unit (C = Celsius) + + Example: + $IIMTW,26.5,C*1D + """ + + talker_id = "II" + sentence_type = "MTW" + + def __init__(self, temp_c: Optional[float] = None): + """Initialize MTW sentence. + + Args: + temp_c: Water temperature in Celsius + """ + self.temp_c = temp_c + + def format_fields(self) -> Optional[str]: + """Format MTW fields.""" + if self.temp_c is None: + return None + return f"{self.temp_c:.1f},C" + + +class MTASentence(NMEASentence): + """MTA - Air Temperature. + + Format: + $IIMTA,T.T,C*CC + + Fields: + 1. Air temperature + 2. Unit (C = Celsius) + + Example: + $IIMTA,24.8,C*0E + + Note: + MTA is not a standard NMEA sentence but is commonly used + as a proprietary extension for air temperature, following + the same format as MTW (water temperature). + + Some devices may use XDR (transducer measurement) for air temp: + $IIXDR,C,24.8,C,AirTemp*XX + """ + + talker_id = "II" + sentence_type = "MTA" + + def __init__(self, temp_c: Optional[float] = None): + """Initialize MTA sentence. + + Args: + temp_c: Air temperature in Celsius + """ + self.temp_c = temp_c + + def format_fields(self) -> Optional[str]: + """Format MTA fields.""" + if self.temp_c is None: + return None + return f"{self.temp_c:.1f},C" diff --git a/axiom-nmea/raymarine_nmea/nmea/sentences/transducer.py b/axiom-nmea/raymarine_nmea/nmea/sentences/transducer.py new file mode 100644 index 0000000..a8c43bf --- /dev/null +++ b/axiom-nmea/raymarine_nmea/nmea/sentences/transducer.py @@ -0,0 +1,215 @@ +""" +Transducer measurement NMEA sentences. + +XDR - Transducer Measurements (generic) + +Used for tanks, batteries, and other sensor data that doesn't have +a dedicated NMEA sentence type. +""" + +from typing import Optional, List, Tuple + +from ..sentence import NMEASentence + + +class XDRSentence(NMEASentence): + """XDR - Transducer Measurements. + + Format: + $IIXDR,T,D.D,U,N,T,D.D,U,N,...*CC + + Each transducer reading has 4 fields: + 1. Transducer type + 2. Data value + 3. Units + 4. Transducer name/ID + + Transducer Types: + A = Angular displacement (degrees) + C = Temperature (Celsius) + D = Depth (meters) + F = Frequency (Hz) + H = Humidity (percent) + I = Current (amps) + N = Force (Newtons) + P = Pressure (Pascals) + R = Flow Rate (liters/second) + S = Salinity (ppt) + T = Tachometer (RPM) + U = Volume (liters) + V = Voltage (volts) + G = Generic (no units) + + Tank Level Examples: + $IIXDR,V,75.2,P,FUEL1,V,68.1,P,FUEL2*XX + (P = percent for tank levels) + + Battery Examples: + $IIXDR,U,26.3,V,HOUSE1,U,27.2,V,HOUSE2*XX + (U is sometimes used for voltage, or V for volts unit) + + Note: + XDR can contain multiple transducer readings in one sentence. + Maximum sentence length is 82 characters, so multiple XDR + sentences may be needed for many sensors. + """ + + talker_id = "II" + sentence_type = "XDR" + + def __init__(self, readings: Optional[List[Tuple[str, float, str, str]]] = None): + """Initialize XDR sentence. + + Args: + readings: List of (type, value, unit, name) tuples + Example: [("V", 75.2, "P", "FUEL1"), ("V", 26.3, "V", "HOUSE1")] + """ + self.readings = readings or [] + + def add_reading( + self, + transducer_type: str, + value: float, + unit: str, + name: str + ) -> None: + """Add a transducer reading. + + Args: + transducer_type: Type code (e.g., "V" for volume, "U" for voltage) + value: Numeric value + unit: Unit code (e.g., "P" for percent, "V" for volts) + name: Transducer name/ID + """ + self.readings.append((transducer_type, value, unit, name)) + + def format_fields(self) -> Optional[str]: + """Format XDR fields.""" + if not self.readings: + return None + + parts = [] + for trans_type, value, unit, name in self.readings: + # Clean the name to be NMEA-safe (no commas, asterisks) + safe_name = name.replace(",", "_").replace("*", "_") + # Use 4 decimal places for pressure (bar ~1.0209), 1 for others + if trans_type == "P" and unit == "B": + parts.append(f"{trans_type},{value:.4f},{unit},{safe_name}") + else: + parts.append(f"{trans_type},{value:.1f},{unit},{safe_name}") + + return ",".join(parts) + + @classmethod + def for_tank( + cls, + tank_id: int, + level_pct: float, + name: Optional[str] = None + ) -> 'XDRSentence': + """Create XDR sentence for a tank level. + + Args: + tank_id: Tank ID number + level_pct: Tank level in percent (0-100) + name: Tank name (uses ID if not provided) + + Returns: + XDRSentence instance + """ + tank_name = name or f"TANK{tank_id}" + xdr = cls() + xdr.add_reading("V", level_pct, "P", tank_name) + return xdr + + @classmethod + def for_battery( + cls, + battery_id: int, + voltage: float, + name: Optional[str] = None + ) -> 'XDRSentence': + """Create XDR sentence for a battery voltage. + + Args: + battery_id: Battery ID number + voltage: Battery voltage in volts + name: Battery name (uses ID if not provided) + + Returns: + XDRSentence instance + """ + battery_name = name or f"BATT{battery_id}" + xdr = cls() + xdr.add_reading("U", voltage, "V", battery_name) + return xdr + + @classmethod + def for_tanks( + cls, + tanks: dict, + names: Optional[dict] = None + ) -> 'XDRSentence': + """Create XDR sentence for multiple tanks. + + Args: + tanks: Dict of tank_id -> level_pct + names: Optional dict of tank_id -> name + + Returns: + XDRSentence instance + """ + names = names or {} + xdr = cls() + for tank_id, level in sorted(tanks.items()): + tank_name = names.get(tank_id, f"TANK{tank_id}") + xdr.add_reading("V", level, "P", tank_name) + return xdr + + @classmethod + def for_batteries( + cls, + batteries: dict, + names: Optional[dict] = None + ) -> 'XDRSentence': + """Create XDR sentence for multiple batteries. + + Args: + batteries: Dict of battery_id -> voltage + names: Optional dict of battery_id -> name + + Returns: + XDRSentence instance + """ + names = names or {} + xdr = cls() + for battery_id, voltage in sorted(batteries.items()): + battery_name = names.get(battery_id, f"BATT{battery_id}") + xdr.add_reading("U", voltage, "V", battery_name) + return xdr + + @classmethod + def for_pressure( + cls, + pressure_mbar: float, + name: str = "Barometer" + ) -> 'XDRSentence': + """Create XDR sentence for barometric pressure. + + Args: + pressure_mbar: Pressure in millibars (hPa) + name: Sensor name (default: "Barometer") + + Returns: + XDRSentence instance + + Note: + Pressure is output in bar (1 bar = 1000 mbar) as per NMEA convention. + Example: 1020.9 mbar = 1.0209 bar + Output: $IIXDR,P,1.0209,B,Barometer*XX + """ + xdr = cls() + # Convert mbar to bar (NMEA standard unit) + pressure_bar = pressure_mbar / 1000.0 + xdr.add_reading("P", pressure_bar, "B", name) + return xdr diff --git a/axiom-nmea/raymarine_nmea/nmea/sentences/wind.py b/axiom-nmea/raymarine_nmea/nmea/sentences/wind.py new file mode 100644 index 0000000..0c32198 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/nmea/sentences/wind.py @@ -0,0 +1,147 @@ +""" +Wind-related NMEA sentences. + +MWV - Wind Speed and Angle +MWD - Wind Direction and Speed +""" + +from typing import Optional + +from ..sentence import NMEASentence + + +class MWVSentence(NMEASentence): + """MWV - Wind Speed and Angle. + + Format: + $IIMWV,A.A,R,S.S,U,A*CC + + Fields: + 1. Wind angle in degrees (0-360) + 2. Reference (R=Relative/Apparent, T=True) + 3. Wind speed + 4. Wind speed units (K=km/h, M=m/s, N=knots, S=statute mph) + 5. Status (A=valid, V=invalid) + + Example: + $IIMWV,045.0,R,12.5,N,A*28 + + Note: + MWV can represent either apparent or true wind: + - Relative (R): Wind angle relative to bow, clockwise + - True (T): True wind direction relative to bow + """ + + talker_id = "II" + sentence_type = "MWV" + + def __init__( + self, + angle: Optional[float] = None, + reference: str = "R", + speed: Optional[float] = None, + speed_units: str = "N", + status: str = "A", + ): + """Initialize MWV sentence. + + Args: + angle: Wind angle in degrees (0-360) + reference: R=Relative/Apparent, T=True + speed: Wind speed + speed_units: K=km/h, M=m/s, N=knots, S=mph + status: A=valid, V=invalid + """ + self.angle = angle + self.reference = reference + self.speed = speed + self.speed_units = speed_units + self.status = status + + def format_fields(self) -> Optional[str]: + """Format MWV fields.""" + # Need at least angle or speed + if self.angle is None and self.speed is None: + return None + + angle_str = f"{self.angle:.1f}" if self.angle is not None else "" + speed_str = f"{self.speed:.1f}" if self.speed is not None else "" + + return ( + f"{angle_str}," + f"{self.reference}," + f"{speed_str}," + f"{self.speed_units}," + f"{self.status}" + ) + + +class MWDSentence(NMEASentence): + """MWD - Wind Direction and Speed. + + Format: + $IIMWD,D.D,T,D.D,M,S.S,N,S.S,M*CC + + Fields: + 1. Wind direction (true) in degrees + 2. T = True + 3. Wind direction (magnetic) in degrees + 4. M = Magnetic + 5. Wind speed in knots + 6. N = Knots + 7. Wind speed in m/s + 8. M = Meters/second + + Example: + $IIMWD,270.0,T,258.0,M,12.5,N,6.4,M*5A + + Note: + MWD provides true wind direction (the direction FROM which wind blows), + expressed as a compass bearing, not relative to the vessel. + """ + + talker_id = "II" + sentence_type = "MWD" + + def __init__( + self, + direction_true: Optional[float] = None, + direction_mag: Optional[float] = None, + speed_kts: Optional[float] = None, + ): + """Initialize MWD sentence. + + Args: + direction_true: True wind direction in degrees (from which wind blows) + direction_mag: Magnetic wind direction in degrees + speed_kts: Wind speed in knots + """ + self.direction_true = direction_true + self.direction_mag = direction_mag + self.speed_kts = speed_kts + + def format_fields(self) -> Optional[str]: + """Format MWD fields.""" + # Need at least direction or speed + if self.direction_true is None and self.speed_kts is None: + return None + + # Format directions + dir_t_str = f"{self.direction_true:.1f}" if self.direction_true is not None else "" + dir_m_str = f"{self.direction_mag:.1f}" if self.direction_mag is not None else "" + + # Format speeds + if self.speed_kts is not None: + spd_kts_str = f"{self.speed_kts:.1f}" + spd_ms = self.speed_kts / 1.94384449 + spd_ms_str = f"{spd_ms:.1f}" + else: + spd_kts_str = "" + spd_ms_str = "" + + return ( + f"{dir_t_str},T," + f"{dir_m_str},M," + f"{spd_kts_str},N," + f"{spd_ms_str},M" + ) diff --git a/axiom-nmea/raymarine_nmea/nmea/server.py b/axiom-nmea/raymarine_nmea/nmea/server.py new file mode 100644 index 0000000..c0b3fe4 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/nmea/server.py @@ -0,0 +1,381 @@ +""" +NMEA TCP Server. + +Provides a TCP server that broadcasts NMEA 0183 sentences to connected clients. +This is useful for feeding navigation apps, charting software, and SignalK. +""" + +import asyncio +import logging +import socket +import threading +from typing import List, Optional, Callable, Set + +from ..data.store import SensorData +from .generator import NMEAGenerator + +logger = logging.getLogger(__name__) + + +class NMEATcpServer: + """TCP server that broadcasts NMEA sentences to connected clients. + + This server accepts TCP connections and broadcasts NMEA 0183 sentences + to all connected clients. It uses asyncio internally for robust handling + of multiple concurrent clients - a slow client won't block others. + + The server publishes ALL available NMEA sentences, which is more data + than what Venus OS can display via D-Bus. This includes: + - GPS: GGA, GLL, RMC + - Navigation: HDG, HDT, VTG, VHW + - Wind: MWV (apparent & true), MWD + - Depth: DPT, DBT + - Temperature: MTW, MTA + - Transducers: XDR (tanks, batteries, pressure) + + Example: + from raymarine_nmea import SensorData, NMEAGenerator + from raymarine_nmea.nmea import NMEATcpServer + + sensor_data = SensorData() + server = NMEATcpServer(sensor_data, port=10110) + server.start() + + # Later, in your update loop: + server.broadcast() + + # When done: + server.stop() + + Thread Safety: + This class is thread-safe. The broadcast() method can be called + from any thread, and client connections are managed safely via + asyncio running in a background thread. + """ + + # Default NMEA TCP port (standard) + DEFAULT_PORT = 10110 + + def __init__( + self, + sensor_data: SensorData, + port: int = DEFAULT_PORT, + generator: Optional[NMEAGenerator] = None, + on_client_connect: Optional[Callable[[str, int], None]] = None, + on_client_disconnect: Optional[Callable[[str, int], None]] = None, + ): + """Initialize the NMEA TCP server. + + Args: + sensor_data: SensorData instance to read values from + port: TCP port to listen on (default: 10110) + generator: NMEAGenerator instance (creates default if None) + on_client_connect: Callback when client connects (addr, port) + on_client_disconnect: Callback when client disconnects (addr, port) + """ + self._sensor_data = sensor_data + self._port = port + self._generator = generator or NMEAGenerator() + + self._on_client_connect = on_client_connect + self._on_client_disconnect = on_client_disconnect + + self._running = False + self._loop: Optional[asyncio.AbstractEventLoop] = None + self._thread: Optional[threading.Thread] = None + self._server: Optional[asyncio.Server] = None + + # Client tracking (accessed from asyncio thread) + self._clients: Set[asyncio.StreamWriter] = set() + self._client_addrs: dict = {} # writer -> (addr, port) + + # Thread-safe counter for client count + self._client_count = 0 + self._client_count_lock = threading.Lock() + + @property + def port(self) -> int: + """Get the TCP port.""" + return self._port + + @property + def client_count(self) -> int: + """Get the number of connected clients.""" + with self._client_count_lock: + return self._client_count + + @property + def is_running(self) -> bool: + """Check if server is running.""" + return self._running + + def start(self) -> bool: + """Start the TCP server. + + Returns: + True if server started successfully, False otherwise + """ + if self._running: + logger.warning("NMEATcpServer already running") + return True + + # Create and start the asyncio event loop in a background thread + self._running = True + + # Use an event to wait for server to be ready + ready_event = threading.Event() + startup_error = [None] # Use list to allow modification in nested function + + def run_loop(): + try: + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + + # Start the server + self._loop.run_until_complete(self._start_server(ready_event, startup_error)) + + # Run until stopped + if self._running and not startup_error[0]: + self._loop.run_forever() + except Exception as e: + startup_error[0] = e + ready_event.set() + finally: + # Cleanup + if self._loop: + try: + self._loop.run_until_complete(self._cleanup()) + except Exception: + pass + self._loop.close() + self._loop = None + + self._thread = threading.Thread( + target=run_loop, + daemon=True, + name="NMEATcpServer-AsyncIO" + ) + self._thread.start() + + # Wait for server to be ready (with timeout) + if not ready_event.wait(timeout=5.0): + logger.error("Timeout waiting for NMEA TCP server to start") + self._running = False + return False + + if startup_error[0]: + logger.error(f"Failed to start NMEA TCP server: {startup_error[0]}") + self._running = False + return False + + logger.info(f"NMEA TCP server listening on port {self._port}") + return True + + async def _start_server(self, ready_event: threading.Event, startup_error: list) -> None: + """Start the asyncio TCP server.""" + try: + self._server = await asyncio.start_server( + self._handle_client, + '', + self._port, + reuse_address=True, + ) + ready_event.set() + except OSError as e: + startup_error[0] = e + ready_event.set() + + async def _handle_client( + self, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter + ) -> None: + """Handle a client connection.""" + addr = writer.get_extra_info('peername') + addr_tuple = (addr[0], addr[1]) if addr else ('unknown', 0) + + # Configure socket for optimal NMEA streaming + sock = writer.get_extra_info('socket') + if sock: + self._configure_socket(sock) + + # Track client + self._clients.add(writer) + self._client_addrs[writer] = addr_tuple + with self._client_count_lock: + self._client_count += 1 + + logger.info(f"NMEA TCP client connected: {addr_tuple[0]}:{addr_tuple[1]}") + + # Callback + if self._on_client_connect: + try: + self._on_client_connect(addr_tuple[0], addr_tuple[1]) + except Exception as e: + logger.debug(f"Client connect callback error: {e}") + + try: + # Keep connection alive until client disconnects or server stops + while self._running: + try: + # Check if client is still connected + data = await asyncio.wait_for(reader.read(1), timeout=5.0) + if not data: + # Client disconnected cleanly + break + except asyncio.TimeoutError: + # No data, but connection still alive + continue + except (ConnectionResetError, BrokenPipeError): + break + except Exception as e: + logger.debug(f"Client {addr_tuple[0]}:{addr_tuple[1]} error: {e}") + finally: + await self._remove_client(writer) + + def _configure_socket(self, sock: socket.socket) -> None: + """Configure client socket for optimal NMEA streaming.""" + # Enable TCP keepalive to detect dead connections + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + + # Platform-specific keepalive settings (Linux) + if hasattr(socket, 'TCP_KEEPIDLE'): + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 10) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 5) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 3) + + # Disable Nagle's algorithm for lower latency + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + async def _remove_client(self, writer: asyncio.StreamWriter) -> None: + """Remove a client from tracking.""" + if writer not in self._clients: + return + + self._clients.discard(writer) + addr = self._client_addrs.pop(writer, None) + + with self._client_count_lock: + self._client_count = max(0, self._client_count - 1) + + try: + writer.close() + await writer.wait_closed() + except Exception: + pass + + if addr: + logger.info(f"NMEA TCP client disconnected: {addr[0]}:{addr[1]}") + + # Callback + if self._on_client_disconnect: + try: + self._on_client_disconnect(addr[0], addr[1]) + except Exception as e: + logger.debug(f"Client disconnect callback error: {e}") + + async def _cleanup(self) -> None: + """Clean up server resources.""" + # Close server + if self._server: + self._server.close() + await self._server.wait_closed() + self._server = None + + # Close all clients + for writer in list(self._clients): + await self._remove_client(writer) + + def stop(self) -> None: + """Stop the TCP server and disconnect all clients.""" + if not self._running: + return + + self._running = False + + # Stop the asyncio event loop + if self._loop and self._loop.is_running(): + self._loop.call_soon_threadsafe(self._loop.stop) + + # Wait for thread to finish + if self._thread and self._thread.is_alive(): + self._thread.join(timeout=5.0) + self._thread = None + + logger.info("NMEA TCP server stopped") + + def broadcast(self) -> int: + """Generate and broadcast NMEA sentences to all connected clients. + + This method generates all NMEA sentences from the current sensor data + and sends them to all connected clients. Each client is sent data + independently, so a slow client won't block others. + + Returns: + Number of sentences broadcast (0 if no clients or no data) + """ + if not self._running or not self._loop: + return 0 + + # Check if we have clients + if self.client_count == 0: + return 0 + + # Generate NMEA sentences + sentences = self._generator.generate_all(self._sensor_data) + if not sentences: + return 0 + + # Encode data + data = ''.join(sentences).encode('ascii') + + # Schedule broadcast on the asyncio event loop + try: + asyncio.run_coroutine_threadsafe( + self._broadcast_async(data), + self._loop + ) + except RuntimeError: + # Loop not running + return 0 + + return len(sentences) + + async def _broadcast_async(self, data: bytes) -> None: + """Broadcast data to all clients asynchronously.""" + if not self._clients: + return + + # Send to all clients concurrently with timeout + async def send_to_client(writer: asyncio.StreamWriter) -> bool: + """Send data to a single client. Returns False if client is dead.""" + try: + writer.write(data) + # Use wait_for to timeout slow clients + await asyncio.wait_for(writer.drain(), timeout=2.0) + return True + except (asyncio.TimeoutError, ConnectionResetError, BrokenPipeError, OSError): + return False + except Exception as e: + logger.debug(f"Send error: {e}") + return False + + # Create tasks for all clients + clients = list(self._clients) + results = await asyncio.gather( + *[send_to_client(writer) for writer in clients], + return_exceptions=True + ) + + # Remove dead clients + for writer, success in zip(clients, results): + if success is False or isinstance(success, Exception): + await self._remove_client(writer) + + def get_client_addresses(self) -> List[tuple]: + """Get list of connected client addresses. + + Returns: + List of (host, port) tuples for all connected clients + """ + return list(self._client_addrs.values()) diff --git a/axiom-nmea/raymarine_nmea/protocol/__init__.py b/axiom-nmea/raymarine_nmea/protocol/__init__.py new file mode 100644 index 0000000..23c2d54 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/protocol/__init__.py @@ -0,0 +1,32 @@ +""" +Protocol module for parsing Raymarine LightHouse protobuf format. +""" + +from .constants import ( + WIRE_VARINT, + WIRE_FIXED64, + WIRE_LENGTH, + WIRE_FIXED32, + HEADER_SIZE, + RAD_TO_DEG, + MS_TO_KTS, + FEET_TO_M, + KELVIN_OFFSET, +) +from .parser import ProtobufParser, ProtoField +from .decoder import RaymarineDecoder + +__all__ = [ + "WIRE_VARINT", + "WIRE_FIXED64", + "WIRE_LENGTH", + "WIRE_FIXED32", + "HEADER_SIZE", + "RAD_TO_DEG", + "MS_TO_KTS", + "FEET_TO_M", + "KELVIN_OFFSET", + "ProtobufParser", + "ProtoField", + "RaymarineDecoder", +] diff --git a/axiom-nmea/raymarine_nmea/protocol/constants.py b/axiom-nmea/raymarine_nmea/protocol/constants.py new file mode 100644 index 0000000..0dd624d --- /dev/null +++ b/axiom-nmea/raymarine_nmea/protocol/constants.py @@ -0,0 +1,391 @@ +""" +Protocol constants for Raymarine LightHouse decoding. + +This module defines the reverse-engineered protobuf schema for Raymarine's +LightHouse network protocol. The protocol uses Google Protocol Buffers over +UDP multicast (226.192.206.102:2565) with a 20-byte proprietary header. + +================================================================================ +PACKET STRUCTURE +================================================================================ + + ┌──────────────────────────────────────────────────────────────────────────┐ + │ Bytes 0-19: Raymarine Header (20 bytes) │ + ├──────────────────────────────────────────────────────────────────────────┤ + │ Bytes 20+: Protobuf Payload (variable length) │ + └──────────────────────────────────────────────────────────────────────────┘ + +================================================================================ +MESSAGE HIERARCHY (Proto-like Schema) +================================================================================ + + LightHousePacket { + 1: DeviceInfo device_info [message] + 2: GpsPosition gps [message] ✓ RELIABLE + 3: HeadingData heading [message] ~ VARIABLE + 5: Navigation navigation [message] ✓ RELIABLE + 7: DepthData depth [message] ~ VARIABLE + 13: WindData wind [message] ~ VARIABLE + 14: EngineData[] engines [repeated] ✓ RELIABLE + 15: EnvironmentData environment [message] ✓ RELIABLE + 16: TankData[] tanks [repeated] ✓ RELIABLE + 20: BatteryData[] house_batteries [repeated] ✓ RELIABLE + } + + GpsPosition (Field 2) { + 1: double latitude [fixed64] // Decimal degrees, -90 to +90 + 2: double longitude [fixed64] // Decimal degrees, -180 to +180 + } + + HeadingData (Field 3) { + 1: float cog [fixed32] // Course over ground (radians) + 2: float heading [fixed32] // Compass heading (radians) + } + + Navigation (Field 5) { + 1: float cog [fixed32] // Course over ground (radians) + 3: float sog [fixed32] // Speed over ground (m/s) + } + + DepthData (Field 7) { + 1: float depth [fixed32] // Depth in meters + } + + WindData (Field 13) { + 4: float twd [fixed32] // True wind direction (radians) + 5: float tws [fixed32] // True wind speed (m/s) + 6: float aws [fixed32] // Apparent wind speed (m/s) + } + + EngineData (Field 14, repeated) { + 1: int32 engine_id [varint] // 0=Port, 1=Starboard + 3: EngineSensors sensors [message] + } + + EngineSensors (Field 14.3) { + 4: float battery_voltage [fixed32] // Volts + } + + EnvironmentData (Field 15) { + 1: float pressure [fixed32] // Barometric pressure (Pascals) + 3: float air_temp [fixed32] // Air temperature (Kelvin) + 9: float water_temp [fixed32] // Water temperature (Kelvin) + } + + TankData (Field 16, repeated) { + 1: int32 tank_id [varint] // Tank identifier (see inference) + 2: int32 status [varint] // Tank type/status flag + 3: float level [fixed32] // Fill percentage (0-100) + } + + BatteryData (Field 20, repeated) { + 1: int32 battery_id [varint] // Battery identifier + 3: float voltage [fixed32] // Volts + } + +================================================================================ +INFERENCE RULES (Field-Presence Logic) +================================================================================ + +The protocol does NOT include explicit message type identifiers. Instead, +the presence or absence of fields determines meaning: + +TANK ID INFERENCE (Field 16): + ┌─────────────────────────────────────────────────────────────────────────┐ + │ If tank_id (16.1) is ABSENT: │ + │ - If status == 5 (WASTE) → tank_id = 100 (Black/Gray Water) │ + │ - If status is ABSENT → tank_id = 2 (Port Fuel) │ + │ │ + │ Port Fuel is the ONLY tank that transmits with neither ID nor status. │ + └─────────────────────────────────────────────────────────────────────────┘ + +ENGINE ID INFERENCE (Field 14): + ┌─────────────────────────────────────────────────────────────────────────┐ + │ If engine_id (14.1) is ABSENT: │ + │ - Default to engine_id = 0 (Port Engine) │ + │ │ + │ Starboard engine explicitly sends engine_id = 1. │ + └─────────────────────────────────────────────────────────────────────────┘ + +================================================================================ +WIRE TYPES (Protobuf Encoding) +================================================================================ + +Protobuf encodes each field with a tag: (field_number << 3) | wire_type + + Type 0 (VARINT): Variable-length integers (IDs, counts, enums) + Type 1 (FIXED64): 8-byte values (doubles for GPS coordinates) + Type 2 (LENGTH): Length-delimited (nested messages, strings) + Type 5 (FIXED32): 4-byte values (floats for angles, speeds, voltages) + +================================================================================ +UNIT CONVENTIONS +================================================================================ + + Angles: Radians (0 to 2π) → Convert with RAD_TO_DEG + Speed: Meters/second → Convert with MS_TO_KTS + Temperature: Kelvin → Subtract KELVIN_OFFSET for Celsius + Pressure: Pascals → Multiply by PA_TO_MBAR for millibars + Depth: Meters → Divide by FEET_TO_M for feet + Voltage: Volts (direct) + Tank Level: Percentage (0-100) + +================================================================================ +""" + +# ============================================================================== +# WIRE TYPES +# ============================================================================== + +WIRE_VARINT = 0 # Variable-length integers (IDs, counts, enums) +WIRE_FIXED64 = 1 # 8-byte values (GPS coordinates as doubles) +WIRE_LENGTH = 2 # Length-delimited (nested messages, strings) +WIRE_FIXED32 = 5 # 4-byte values (angles, speeds, voltages as floats) + +# Raymarine packet header size (bytes before protobuf payload) +HEADER_SIZE = 20 + + +# ============================================================================== +# TOP-LEVEL FIELDS +# ============================================================================== + +class Fields: + """Top-level protobuf field numbers in LightHousePacket. + + All top-level fields are length-delimited messages (wire type 2). + Fields marked [repeated] appear multiple times for multiple instances. + """ + DEVICE_INFO = 1 # DeviceInfo - Device name and serial + GPS_POSITION = 2 # GpsPosition - Latitude/longitude (reliable) + HEADING = 3 # HeadingData - Compass heading (variable) + SOG_COG = 5 # Navigation - Speed/course over ground (reliable) + DEPTH = 7 # DepthData - Water depth (variable, large packets) + WIND_NAVIGATION = 13 # WindData - Wind speed/direction (variable) + ENGINE_DATA = 14 # EngineData[] - [repeated] Engine sensors + battery + TEMPERATURE = 15 # EnvironmentData - Temp/pressure (reliable) + TANK_DATA = 16 # TankData[] - [repeated] Tank levels (reliable) + HOUSE_BATTERY = 20 # BatteryData[] - [repeated] House batteries (reliable) + + +# ============================================================================== +# NESTED FIELD DEFINITIONS +# ============================================================================== + +class GPSFields: + """Field 2: GpsPosition - GPS coordinates. + + Reliability: ✓ HIGH + Wire types: All fields are fixed64 (8-byte doubles) + + Example values: + latitude: 26.123456 (decimal degrees) + longitude: -80.654321 (decimal degrees) + """ + LATITUDE = 1 # fixed64/double - Decimal degrees, -90 to +90 + LONGITUDE = 2 # fixed64/double - Decimal degrees, -180 to +180 + COG_RAD = 3 # fixed64/double - Course over ground (radians), NaN when stationary + SOG_MS = 4 # fixed32/float - Speed over ground (m/s) + + +class HeadingFields: + """Field 3: HeadingData - Compass and course data. + + Reliability: ~ VARIABLE (context-dependent) + Wire types: All fields are fixed32 (4-byte floats) + """ + COG_RAD = 1 # fixed32/float - Course over ground (radians) + HEADING_RAD = 2 # fixed32/float - Compass heading (radians, 0 to 2π) + + +class SOGCOGFields: + """Field 5: Navigation - GPS-derived speed and course. + + Reliability: ✓ HIGH + Wire types: All fields are fixed32 (4-byte floats) + Packet size: Found in 86-92 byte packets + + This is the PRIMARY source for SOG/COG data. + """ + COG_RAD = 1 # fixed32/float - Course over ground (radians) + # Field 2 exists but purpose unknown + SOG_MS = 3 # fixed32/float - Speed over ground (m/s) + + +class DepthFields: + """Field 7: DepthData - Water depth. + + Reliability: ~ VARIABLE (only in larger packets) + Wire types: fixed32 (4-byte float) + """ + DEPTH_METERS = 1 # fixed32/float - Depth in meters + + +class WindFields: + """Field 13: WindData - Wind speed and direction. + + Reliability: ~ VARIABLE (depends on wind sensor availability) + Wire types: All fields are fixed32 (4-byte floats) + + Note: Fields 1-3 exist but purpose unknown. + """ + # Fields 1-3 unknown + TRUE_WIND_DIRECTION = 4 # fixed32/float - TWD in radians + TRUE_WIND_SPEED = 5 # fixed32/float - TWS in m/s + APPARENT_WIND_SPEED = 6 # fixed32/float - AWS in m/s + + +class TemperatureFields: + """Field 15: EnvironmentData - Temperature and pressure sensors. + + Reliability: ✓ HIGH + Wire types: All fields are fixed32 (4-byte floats) + + Note: Fields 2, 4-8 exist but purpose unknown. + """ + BAROMETRIC_PRESSURE = 1 # fixed32/float - Pascals (divide by 100 for mbar) + # Field 2 unknown + AIR_TEMP = 3 # fixed32/float - Kelvin (subtract 273.15 for °C) + # Fields 4-8 unknown + WATER_TEMP = 9 # fixed32/float - Kelvin (subtract 273.15 for °C) + + +class TankFields: + """Field 16: TankData - Tank level sensors (repeated message). + + Reliability: ✓ HIGH + Wire types: Mixed (see individual fields) + + INFERENCE RULE: + If tank_id is ABSENT: + - status == 5 (WASTE) → tank_id = 100 (Black/Gray Water) + - status is ABSENT → tank_id = 2 (Port Fuel, unique case) + """ + TANK_ID = 1 # varint/int32 - Tank identifier (may be absent, see inference) + STATUS = 2 # varint/int32 - Tank type flag (5 = waste tank) + LEVEL_PCT = 3 # fixed32/float - Fill percentage (0-100) + + +class BatteryFields: + """Field 20: BatteryData - House battery sensors (repeated message). + + Reliability: ✓ HIGH + Wire types: Mixed (see individual fields) + + Known battery IDs: 11, 13 (house batteries) + """ + BATTERY_ID = 1 # varint/int32 - Battery identifier + # Field 2 unknown + VOLTAGE = 3 # fixed32/float - Voltage in volts + + +class EngineFields: + """Field 14: EngineData - Engine sensors (repeated message). + + Reliability: ✓ HIGH + Wire types: Mixed (see individual fields) + + Structure: 3 levels of nesting + Field 14 (EngineData) + └─ Field 14.3 (EngineSensors message) + └─ Field 14.3.4 (battery_voltage float) + + INFERENCE RULE: + If engine_id is ABSENT → default to 0 (Port Engine) + Starboard engine explicitly sends engine_id = 1 + + Battery IDs: Stored as 1000 + engine_id (1000=Port, 1001=Starboard) + """ + ENGINE_ID = 1 # varint/int32 - Engine ID (0=Port, 1=Starboard) + # Field 2 unknown + SENSOR_DATA = 3 # message - EngineSensors (nested message) + + # Within SENSOR_DATA (Field 14.3): + # Fields 1-3 unknown + BATTERY_VOLTAGE = 4 # fixed32/float - Battery voltage in volts + +# ============================================================================== +# UNIT CONVERSIONS +# ============================================================================== +# Raymarine uses SI units internally. These convert to common marine units. + +RAD_TO_DEG = 57.2957795131 # radians → degrees (180/π) +MS_TO_KTS = 1.94384449 # m/s → knots +FEET_TO_M = 0.3048 # feet → meters +KELVIN_OFFSET = 273.15 # Kelvin → Celsius (subtract) +FATHOMS_TO_M = 1.8288 # fathoms → meters +PA_TO_MBAR = 0.01 # Pascals → millibars (hPa) + + +# ============================================================================== +# VALIDATION RANGES +# ============================================================================== +# Values outside these ranges are rejected as invalid/corrupt data. + +class ValidationRanges: + """Valid ranges for decoded sensor values. + + These ranges filter out corrupt or invalid data. Values outside + these bounds are discarded during decoding. + """ + + # ------------------------------------------------------------------------- + # GPS Position + # ------------------------------------------------------------------------- + LATITUDE_MIN = -90.0 + LATITUDE_MAX = 90.0 + LONGITUDE_MIN = -180.0 + LONGITUDE_MAX = 180.0 + NULL_ISLAND_THRESHOLD = 0.1 # Reject positions within 0.1° of (0,0) + + # ------------------------------------------------------------------------- + # Angles (radians) - Heading, COG, Wind Direction + # ------------------------------------------------------------------------- + ANGLE_MIN = 0.0 + ANGLE_MAX = 6.5 # Slightly > 2π (6.283) to allow small errors + + # ------------------------------------------------------------------------- + # Speed (m/s) - SOG, Wind Speed + # ------------------------------------------------------------------------- + SPEED_MIN = 0.0 + SPEED_MAX = 100.0 # ~194 knots (theoretical max for any vessel) + + # ------------------------------------------------------------------------- + # Depth (meters) + # ------------------------------------------------------------------------- + DEPTH_MIN = 0.0 + DEPTH_MAX = 1000.0 # ~3280 feet / ~546 fathoms + + # ------------------------------------------------------------------------- + # Temperature (Kelvin) + # ------------------------------------------------------------------------- + AIR_TEMP_MIN = 200.0 # -73°C (extreme cold) + AIR_TEMP_MAX = 350.0 # +77°C (extreme heat) + WATER_TEMP_MIN = 270.0 # -3°C (near freezing) + WATER_TEMP_MAX = 320.0 # +47°C (tropical/engine room) + + # ------------------------------------------------------------------------- + # Tank Level (percentage) + # ------------------------------------------------------------------------- + TANK_MIN = 0.0 + TANK_MAX = 100.0 + + # ------------------------------------------------------------------------- + # Battery Voltage + # ------------------------------------------------------------------------- + VOLTAGE_MIN = 10.0 # Below 10V = dead/disconnected + VOLTAGE_MAX = 60.0 # Covers 12V, 24V, 48V systems with headroom + + # ------------------------------------------------------------------------- + # Barometric Pressure (Pascals) + # ------------------------------------------------------------------------- + PRESSURE_MIN = 87000.0 # ~870 mbar (record low: 870 mbar, Typhoon Tip) + PRESSURE_MAX = 108400.0 # ~1084 mbar (record high: 1084 mbar, Siberia) + + +# ============================================================================== +# PROTOCOL CONSTANTS +# ============================================================================== + +# Tank status values (Field 16.2) +TANK_STATUS_WASTE = 5 # Black/gray water tanks transmit status=5 diff --git a/axiom-nmea/raymarine_nmea/protocol/decoder.py b/axiom-nmea/raymarine_nmea/protocol/decoder.py new file mode 100644 index 0000000..d24d477 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/protocol/decoder.py @@ -0,0 +1,516 @@ +""" +Raymarine packet decoder. + +Decodes Raymarine LightHouse protobuf packets and extracts sensor data +into structured Python objects. +""" + +from typing import Dict, List, Optional, Any +from dataclasses import dataclass, field as dc_field +import time + +from .parser import ProtobufParser, ProtoField +from .constants import ( + WIRE_VARINT, + WIRE_FIXED64, + WIRE_LENGTH, + WIRE_FIXED32, + HEADER_SIZE, + RAD_TO_DEG, + MS_TO_KTS, + FEET_TO_M, + KELVIN_OFFSET, + PA_TO_MBAR, + Fields, + GPSFields, + HeadingFields, + SOGCOGFields, + DepthFields, + WindFields, + TemperatureFields, + TankFields, + BatteryFields, + EngineFields, + ValidationRanges, + TANK_STATUS_WASTE, +) + + +@dataclass +class DecodedData: + """Container for decoded sensor values from a single packet. + + This is a lightweight container for data extracted from one packet. + For aggregated data across multiple packets, use SensorData. + """ + # Position + latitude: Optional[float] = None + longitude: Optional[float] = None + + # Navigation + heading_deg: Optional[float] = None + cog_deg: Optional[float] = None + sog_kts: Optional[float] = None + + # Wind + twd_deg: Optional[float] = None # True Wind Direction + tws_kts: Optional[float] = None # True Wind Speed + awa_deg: Optional[float] = None # Apparent Wind Angle + aws_kts: Optional[float] = None # Apparent Wind Speed + + # Depth + depth_m: Optional[float] = None + + # Temperature + water_temp_c: Optional[float] = None + air_temp_c: Optional[float] = None + + # Barometric pressure + pressure_mbar: Optional[float] = None + + # Tanks: dict of tank_id -> level percentage + tanks: Dict[int, float] = dc_field(default_factory=dict) + + # Batteries: dict of battery_id -> voltage + batteries: Dict[int, float] = dc_field(default_factory=dict) + + # Decode timestamp + timestamp: float = dc_field(default_factory=time.time) + + def has_data(self) -> bool: + """Check if any data was decoded.""" + return ( + self.latitude is not None or + self.longitude is not None or + self.heading_deg is not None or + self.cog_deg is not None or + self.sog_kts is not None or + self.twd_deg is not None or + self.tws_kts is not None or + self.depth_m is not None or + self.water_temp_c is not None or + self.air_temp_c is not None or + self.pressure_mbar is not None or + bool(self.tanks) or + bool(self.batteries) + ) + + +class RaymarineDecoder: + """Decodes Raymarine packets using proper protobuf parsing. + + This decoder implements field-based parsing of Raymarine's protobuf + protocol. It extracts all supported sensor types and validates values. + + Example: + decoder = RaymarineDecoder() + result = decoder.decode(packet_bytes) + if result.latitude: + print(f"GPS: {result.latitude}, {result.longitude}") + """ + + def __init__(self, verbose: bool = False): + """Initialize the decoder. + + Args: + verbose: If True, print decoded values + """ + self.verbose = verbose + + def decode(self, packet: bytes) -> DecodedData: + """Decode a single Raymarine packet. + + Args: + packet: Raw packet bytes (including 20-byte header) + + Returns: + DecodedData containing all extracted values + """ + result = DecodedData() + + # Need at least header + some protobuf data + if len(packet) < HEADER_SIZE + 20: + return result + + # Skip fixed header, protobuf starts at offset 0x14 + proto_data = packet[HEADER_SIZE:] + + # Parse protobuf - collect repeated fields: + # Field 14 = Engine data (contains battery voltage at 14.3.4) + # Field 16 = Tank data + # Field 20 = House battery data + parser = ProtobufParser(proto_data) + fields = parser.parse_message(collect_repeated={14, 16, 20}) + + if not fields: + return result + + # Extract GPS from Field 2 + if Fields.GPS_POSITION in fields: + gps_field = fields[Fields.GPS_POSITION] + if gps_field.children: + self._extract_gps(gps_field.children, result) + + # Extract Heading from Field 3 + if Fields.HEADING in fields: + heading_field = fields[Fields.HEADING] + if heading_field.children: + self._extract_heading(heading_field.children, result) + + # Extract SOG/COG from Field 5 (primary source for SOG/COG) + if Fields.SOG_COG in fields: + sog_cog_field = fields[Fields.SOG_COG] + if sog_cog_field.children: + self._extract_sog_cog(sog_cog_field.children, result) + + # Extract Wind from Field 13 + if Fields.WIND_NAVIGATION in fields: + wind_field = fields[Fields.WIND_NAVIGATION] + if wind_field.children: + self._extract_wind(wind_field.children, result) + + # Extract Depth from Field 7 (only in larger packets) + if Fields.DEPTH in fields: + depth_field = fields[Fields.DEPTH] + if depth_field.children: + self._extract_depth(depth_field.children, result) + + # Extract Temperature from Field 15 + if Fields.TEMPERATURE in fields: + temp_field = fields[Fields.TEMPERATURE] + if temp_field.children: + self._extract_temperature(temp_field.children, result) + + # Extract Tank data from Field 16 (repeated) + if Fields.TANK_DATA in fields: + tank_fields = fields[Fields.TANK_DATA] # This is a list + self._extract_tanks(tank_fields, result) + + # Extract Battery data from Field 20 (repeated) - house batteries + if Fields.HOUSE_BATTERY in fields: + battery_fields = fields[Fields.HOUSE_BATTERY] # This is a list + self._extract_batteries(battery_fields, result) + + # Extract Engine battery data from Field 14 (repeated) + if Fields.ENGINE_DATA in fields: + engine_fields = fields[Fields.ENGINE_DATA] # This is a list + self._extract_engine_batteries(engine_fields, result) + + return result + + def _extract_gps( + self, + fields: Dict[int, ProtoField], + result: DecodedData + ) -> None: + """Extract GPS position from Field 2's children.""" + lat = None + lon = None + + # Field 1 = Latitude + if GPSFields.LATITUDE in fields: + f = fields[GPSFields.LATITUDE] + if f.wire_type == WIRE_FIXED64: + lat = ProtobufParser.decode_double(f.value) + + # Field 2 = Longitude + if GPSFields.LONGITUDE in fields: + f = fields[GPSFields.LONGITUDE] + if f.wire_type == WIRE_FIXED64: + lon = ProtobufParser.decode_double(f.value) + + # Validate lat/lon + if lat is not None and lon is not None: + if (ValidationRanges.LATITUDE_MIN <= lat <= ValidationRanges.LATITUDE_MAX and + ValidationRanges.LONGITUDE_MIN <= lon <= ValidationRanges.LONGITUDE_MAX): + # Check not at null island + if (abs(lat) > ValidationRanges.NULL_ISLAND_THRESHOLD or + abs(lon) > ValidationRanges.NULL_ISLAND_THRESHOLD): + result.latitude = lat + result.longitude = lon + if self.verbose: + print(f"GPS: {lat:.6f}, {lon:.6f}") + + def _extract_heading( + self, + fields: Dict[int, ProtoField], + result: DecodedData + ) -> None: + """Extract heading from Field 3's children.""" + # Field 2 = Heading in radians + if HeadingFields.HEADING_RAD in fields: + f = fields[HeadingFields.HEADING_RAD] + if f.wire_type == WIRE_FIXED32: + val = ProtobufParser.decode_float(f.value) + if val is not None: + if ValidationRanges.ANGLE_MIN <= val <= ValidationRanges.ANGLE_MAX: + heading_deg = (val * RAD_TO_DEG) % 360 + result.heading_deg = heading_deg + if self.verbose: + print(f"Heading: {heading_deg:.1f}°") + + def _extract_sog_cog( + self, + fields: Dict[int, ProtoField], + result: DecodedData + ) -> None: + """Extract SOG and COG from Field 5's children. + + Field 5 contains GPS-derived navigation data. + Field 5.1 = COG (shows most variation in real data) + Field 5.3 = SOG (confirmed with real data) + """ + # Field 5.1 = COG (Course Over Ground) in radians - confirmed with real data + if SOGCOGFields.COG_RAD in fields: + f = fields[SOGCOGFields.COG_RAD] + if f.wire_type == WIRE_FIXED32: + val = ProtobufParser.decode_float(f.value) + if val is not None: + if ValidationRanges.ANGLE_MIN <= val <= ValidationRanges.ANGLE_MAX: + cog_deg = (val * RAD_TO_DEG) % 360 + result.cog_deg = cog_deg + if self.verbose: + print(f"COG: {cog_deg:.1f}°") + + # Field 5.3 = SOG (Speed Over Ground) in m/s - confirmed with real data + if SOGCOGFields.SOG_MS in fields: + f = fields[SOGCOGFields.SOG_MS] + if f.wire_type == WIRE_FIXED32: + val = ProtobufParser.decode_float(f.value) + if val is not None: + if ValidationRanges.SPEED_MIN <= val <= ValidationRanges.SPEED_MAX: + sog_kts = val * MS_TO_KTS + result.sog_kts = sog_kts + if self.verbose: + print(f"SOG: {sog_kts:.1f} kts") + + def _extract_wind( + self, + fields: Dict[int, ProtoField], + result: DecodedData + ) -> None: + """Extract wind data from Field 13's children.""" + # Field 4 = True Wind Direction (radians) + if WindFields.TRUE_WIND_DIRECTION in fields: + f = fields[WindFields.TRUE_WIND_DIRECTION] + if f.wire_type == WIRE_FIXED32: + val = ProtobufParser.decode_float(f.value) + if val is not None: + if ValidationRanges.ANGLE_MIN <= val <= ValidationRanges.ANGLE_MAX: + twd_deg = (val * RAD_TO_DEG) % 360 + result.twd_deg = twd_deg + if self.verbose: + print(f"TWD: {twd_deg:.1f}°") + + # Field 5 = True Wind Speed (m/s) + if WindFields.TRUE_WIND_SPEED in fields: + f = fields[WindFields.TRUE_WIND_SPEED] + if f.wire_type == WIRE_FIXED32: + val = ProtobufParser.decode_float(f.value) + if val is not None: + if ValidationRanges.SPEED_MIN <= val <= ValidationRanges.SPEED_MAX: + tws_kts = val * MS_TO_KTS + result.tws_kts = tws_kts + if self.verbose: + print(f"TWS: {tws_kts:.1f} kts") + + # Field 6 = Apparent Wind Speed (m/s) + if WindFields.APPARENT_WIND_SPEED in fields: + f = fields[WindFields.APPARENT_WIND_SPEED] + if f.wire_type == WIRE_FIXED32: + val = ProtobufParser.decode_float(f.value) + if val is not None: + if ValidationRanges.SPEED_MIN <= val <= ValidationRanges.SPEED_MAX: + aws_kts = val * MS_TO_KTS + result.aws_kts = aws_kts + if self.verbose: + print(f"AWS: {aws_kts:.1f} kts") + + def _extract_depth( + self, + fields: Dict[int, ProtoField], + result: DecodedData + ) -> None: + """Extract depth from Field 7's children.""" + if DepthFields.DEPTH_METERS in fields: + f = fields[DepthFields.DEPTH_METERS] + if f.wire_type == WIRE_FIXED32: + val = ProtobufParser.decode_float(f.value) + if val is not None: + if ValidationRanges.DEPTH_MIN < val <= ValidationRanges.DEPTH_MAX: + result.depth_m = val + if self.verbose: + depth_ft = val / FEET_TO_M + print(f"Depth: {depth_ft:.1f} ft ({val:.2f} m)") + + def _extract_temperature( + self, + fields: Dict[int, ProtoField], + result: DecodedData + ) -> None: + """Extract temperature and pressure from Field 15's children.""" + # Field 1 = Barometric Pressure (Pascals) + if TemperatureFields.BAROMETRIC_PRESSURE in fields: + f = fields[TemperatureFields.BAROMETRIC_PRESSURE] + if f.wire_type == WIRE_FIXED32: + val = ProtobufParser.decode_float(f.value) + if val is not None: + if ValidationRanges.PRESSURE_MIN <= val <= ValidationRanges.PRESSURE_MAX: + pressure_mbar = val * PA_TO_MBAR + result.pressure_mbar = pressure_mbar + if self.verbose: + print(f"Pressure: {pressure_mbar:.1f} mbar") + + # Field 3 = Air Temperature (Kelvin) + if TemperatureFields.AIR_TEMP in fields: + f = fields[TemperatureFields.AIR_TEMP] + if f.wire_type == WIRE_FIXED32: + val = ProtobufParser.decode_float(f.value) + if val is not None: + if ValidationRanges.AIR_TEMP_MIN <= val <= ValidationRanges.AIR_TEMP_MAX: + temp_c = val - KELVIN_OFFSET + result.air_temp_c = temp_c + if self.verbose: + print(f"Air Temp: {temp_c:.1f}°C") + + # Field 9 = Water Temperature (Kelvin) + if TemperatureFields.WATER_TEMP in fields: + f = fields[TemperatureFields.WATER_TEMP] + if f.wire_type == WIRE_FIXED32: + val = ProtobufParser.decode_float(f.value) + if val is not None: + if ValidationRanges.WATER_TEMP_MIN <= val <= ValidationRanges.WATER_TEMP_MAX: + temp_c = val - KELVIN_OFFSET + result.water_temp_c = temp_c + if self.verbose: + print(f"Water Temp: {temp_c:.1f}°C") + + def _extract_tanks( + self, + tank_fields: List[ProtoField], + result: DecodedData + ) -> None: + """Extract tank levels from Field 16 (repeated).""" + for tank_field in tank_fields: + if not tank_field.children: + continue + + children = tank_field.children + tank_id = None + level = None + status = None + + # Field 1 = Tank ID (varint) + if TankFields.TANK_ID in children: + f = children[TankFields.TANK_ID] + if f.wire_type == WIRE_VARINT: + tank_id = f.value + + # Field 2 = Status (varint) + if TankFields.STATUS in children: + f = children[TankFields.STATUS] + if f.wire_type == WIRE_VARINT: + status = f.value + + # Field 3 = Tank Level percentage (float) + if TankFields.LEVEL_PCT in children: + f = children[TankFields.LEVEL_PCT] + if f.wire_type == WIRE_FIXED32: + val = ProtobufParser.decode_float(f.value) + if val is not None: + if ValidationRanges.TANK_MIN <= val <= ValidationRanges.TANK_MAX: + level = val + + # If we have a level but no tank_id, try to infer it + if tank_id is None and level is not None: + if status == TANK_STATUS_WASTE: + # Black/gray water tank + tank_id = 100 + elif status is None: + # Port Fuel is the ONLY tank with neither ID nor status + tank_id = 2 + + if tank_id is not None and level is not None: + result.tanks[tank_id] = level + if self.verbose: + print(f"Tank {tank_id}: {level:.1f}%") + + def _extract_batteries( + self, + battery_fields: List[ProtoField], + result: DecodedData + ) -> None: + """Extract battery voltages from Field 20 (repeated).""" + for battery_field in battery_fields: + if not battery_field.children: + continue + + children = battery_field.children + battery_id = None + voltage = None + + # Field 1 = Battery ID (varint) + if BatteryFields.BATTERY_ID in children: + f = children[BatteryFields.BATTERY_ID] + if f.wire_type == WIRE_VARINT: + battery_id = f.value + + # Field 3 = Voltage (float) + if BatteryFields.VOLTAGE in children: + f = children[BatteryFields.VOLTAGE] + if f.wire_type == WIRE_FIXED32: + val = ProtobufParser.decode_float(f.value) + if val is not None: + if ValidationRanges.VOLTAGE_MIN <= val <= ValidationRanges.VOLTAGE_MAX: + voltage = val + + if battery_id is not None and voltage is not None: + result.batteries[battery_id] = voltage + if self.verbose: + print(f"Battery {battery_id}: {voltage:.2f}V") + + def _extract_engine_batteries( + self, + engine_fields: List[ProtoField], + result: DecodedData + ) -> None: + """Extract engine battery voltages from Field 14 (repeated). + + Engine data structure: + Field 14.1 (varint): Engine ID (0=Port, 1=Starboard) + Field 14.3 (message): Engine sensor data + Field 14.3.4 (float): Battery voltage + """ + for engine_field in engine_fields: + if not engine_field.children: + continue + + children = engine_field.children + + # Field 1 = Engine ID (varint), default 0 (Port) if not present + engine_id = 0 + if EngineFields.ENGINE_ID in children: + f = children[EngineFields.ENGINE_ID] + if f.wire_type == WIRE_VARINT: + engine_id = f.value + + # Field 3 = Engine sensor data (nested message) + if EngineFields.SENSOR_DATA in children: + f = children[EngineFields.SENSOR_DATA] + if f.wire_type == WIRE_LENGTH: + sensor_data = f.value + # Parse the nested message to get Field 4 (voltage) + sensor_parser = ProtobufParser(sensor_data) + sensor_fields = sensor_parser.parse_message() + + if EngineFields.BATTERY_VOLTAGE in sensor_fields: + vf = sensor_fields[EngineFields.BATTERY_VOLTAGE] + if vf.wire_type == WIRE_FIXED32: + val = ProtobufParser.decode_float(vf.value) + if val is not None: + if ValidationRanges.VOLTAGE_MIN <= val <= ValidationRanges.VOLTAGE_MAX: + # Use battery_id = 1000 + engine_id + battery_id = 1000 + engine_id + result.batteries[battery_id] = val + if self.verbose: + print(f"Engine {engine_id} Battery: {val:.2f}V") diff --git a/axiom-nmea/raymarine_nmea/protocol/parser.py b/axiom-nmea/raymarine_nmea/protocol/parser.py new file mode 100644 index 0000000..1f70521 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/protocol/parser.py @@ -0,0 +1,243 @@ +""" +Protobuf wire format parser for Raymarine packets. + +This parser implements the Google Protocol Buffers wire format without +requiring a schema (.proto file). It can parse any protobuf message and +return the field structure. + +Wire Format Reference: + https://developers.google.com/protocol-buffers/docs/encoding +""" + +import struct +from dataclasses import dataclass, field +from typing import Any, Dict, Optional, Set + +from .constants import ( + WIRE_VARINT, + WIRE_FIXED64, + WIRE_LENGTH, + WIRE_FIXED32, +) + + +@dataclass +class ProtoField: + """A decoded protobuf field. + + Attributes: + field_num: The field number (1-536870911) + wire_type: The wire type (0, 1, 2, or 5) + value: The decoded value (int, bytes, or float) + children: For length-delimited fields, nested message fields + """ + field_num: int + wire_type: int + value: Any + children: Dict[int, 'ProtoField'] = field(default_factory=dict) + + +class ProtobufParser: + """Parses protobuf wire format without a schema. + + This parser reads raw protobuf data and extracts fields based on their + wire type. For length-delimited fields, it attempts to parse them as + nested messages. + + Example: + parser = ProtobufParser(packet_bytes) + fields = parser.parse_message(collect_repeated={14, 16, 20}) + if 2 in fields: + gps_field = fields[2] + if gps_field.children: + # Access nested GPS fields + lat_field = gps_field.children.get(1) + """ + + def __init__(self, data: bytes): + """Initialize parser with protobuf data. + + Args: + data: Raw protobuf bytes to parse + """ + self.data = data + self.pos = 0 + + def remaining(self) -> int: + """Return number of unread bytes.""" + return len(self.data) - self.pos + + def read_varint(self) -> int: + """Decode a variable-length integer. + + Varints use 7 bits per byte for the value, with the high bit + indicating whether more bytes follow. + + Returns: + The decoded integer value + + Raises: + IndexError: If data ends before varint is complete + """ + result = 0 + shift = 0 + while self.pos < len(self.data): + byte = self.data[self.pos] + self.pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + if shift > 63: + break + return result + + def read_fixed64(self) -> bytes: + """Read 8 bytes (fixed64 wire type). + + Returns: + 8 bytes of raw data + """ + value = self.data[self.pos:self.pos + 8] + self.pos += 8 + return value + + def read_fixed32(self) -> bytes: + """Read 4 bytes (fixed32 wire type). + + Returns: + 4 bytes of raw data + """ + value = self.data[self.pos:self.pos + 4] + self.pos += 4 + return value + + def read_length_delimited(self) -> bytes: + """Read length-prefixed data. + + First reads a varint for the length, then reads that many bytes. + + Returns: + The length-delimited data + """ + length = self.read_varint() + value = self.data[self.pos:self.pos + length] + self.pos += length + return value + + def parse_message( + self, + collect_repeated: Optional[Set[int]] = None + ) -> Dict[int, Any]: + """Parse all fields in a protobuf message. + + Args: + collect_repeated: Set of field numbers to collect as lists. + Use this for repeated fields like tanks, batteries. + + Returns: + Dictionary mapping field numbers to ProtoField objects. + For repeated fields (in collect_repeated), maps to list of ProtoField. + """ + fields: Dict[int, Any] = {} + if collect_repeated is None: + collect_repeated = set() + + while self.pos < len(self.data): + if self.remaining() < 1: + break + + try: + # Read tag: (field_number << 3) | wire_type + tag = self.read_varint() + field_num = tag >> 3 + wire_type = tag & 0x07 + + # Validate field number + if field_num == 0 or field_num > 536870911: + break + + # Read value based on wire type + if wire_type == WIRE_VARINT: + value = self.read_varint() + elif wire_type == WIRE_FIXED64: + if self.remaining() < 8: + break + value = self.read_fixed64() + elif wire_type == WIRE_LENGTH: + value = self.read_length_delimited() + elif wire_type == WIRE_FIXED32: + if self.remaining() < 4: + break + value = self.read_fixed32() + else: + # Unknown wire type (3, 4 are deprecated) + break + + # For length-delimited, try to parse as nested message + children: Dict[int, ProtoField] = {} + if wire_type == WIRE_LENGTH and len(value) >= 2: + try: + nested_parser = ProtobufParser(value) + children = nested_parser.parse_message() + # Only keep if we parsed most of the data + if nested_parser.pos < len(value) * 0.5: + children = {} + except Exception: + children = {} + + pf = ProtoField(field_num, wire_type, value, children) + + # Handle repeated fields - collect as list + if field_num in collect_repeated: + if field_num not in fields: + fields[field_num] = [] + fields[field_num].append(pf) + else: + # Keep last occurrence for non-repeated fields + fields[field_num] = pf + + except (IndexError, struct.error): + break + + return fields + + @staticmethod + def decode_double(raw: bytes) -> Optional[float]: + """Decode 8 bytes as little-endian double. + + Args: + raw: 8 bytes of raw data + + Returns: + Decoded float value, or None if invalid/NaN + """ + if len(raw) != 8: + return None + try: + val = struct.unpack(' Optional[float]: + """Decode 4 bytes as little-endian float. + + Args: + raw: 4 bytes of raw data + + Returns: + Decoded float value, or None if invalid/NaN + """ + if len(raw) != 4: + return None + try: + val = struct.unpack(' Optional[float]: + """Return capacity in liters.""" + if self.capacity_gallons is None: + return None + return self.capacity_gallons * 3.78541 + + +@dataclass +class BatteryInfo: + """Configuration for a battery sensor.""" + name: str + nominal_voltage: float = 12.0 # 12V, 24V, 48V + battery_type: str = "house" # house, engine, starter + + +# Default tank configuration +# Key is the tank ID from Raymarine +TANK_CONFIG: Dict[int, TankInfo] = { + 1: TankInfo("Fuel Starboard", 265, "fuel"), + 2: TankInfo("Fuel Port", 265, "fuel"), + 10: TankInfo("Water Bow", 90, "water"), + 11: TankInfo("Water Stern", 90, "water"), + 100: TankInfo("Black Water", 53, "blackwater"), +} + +# Default battery configuration +# Key is the battery ID from Raymarine +# IDs 1000+ are engine batteries (1000 + engine_id) +BATTERY_CONFIG: Dict[int, BatteryInfo] = { + 11: BatteryInfo("House Battery Bow", 24.0, "house"), + 13: BatteryInfo("House Battery Stern", 24.0, "house"), + 1000: BatteryInfo("Engine Port", 24.0, "engine"), + 1001: BatteryInfo("Engine Starboard", 24.0, "engine"), +} + +# Raymarine multicast groups +# Each tuple is (group_address, port) +# +# OPTIMIZATION: Based on testing, ALL sensor data (GPS, wind, depth, heading, +# temperature, tanks, batteries) comes from the primary group. The other groups +# contain only heartbeats, display sync, or zero/empty data. +# +# Test results showed: +# 226.192.206.102:2565 - 520 pkts, 297 decoded (GPS, heading, wind, depth, temp, tanks, batteries) +# 226.192.219.0:3221 - 2707 pkts, 0 decoded (display sync - high traffic, no data!) +# 226.192.206.99:2562 - 402 pkts, 0 decoded (heartbeat only) +# 226.192.206.98:2561 - 356 pkts, 0 decoded (mostly zeros) +# Others - <30 pkts each, 0 decoded +# +# Using only the primary group reduces: +# - Thread count: 7 → 1 (less context switching) +# - Packet processing: ~4000 → ~500 packets (87% reduction) +# - CPU usage: significant reduction on embedded devices + +# Primary multicast group - contains ALL sensor data +MULTICAST_GROUPS: list[Tuple[str, int]] = [ + ("226.192.206.102", 2565), # PRIMARY sensor data (GPS, wind, depth, tanks, batteries, etc.) +] + +# Legacy: All groups (for debugging/testing only) +MULTICAST_GROUPS_ALL: list[Tuple[str, int]] = [ + ("226.192.206.98", 2561), # Navigation sensors (mostly zeros) + ("226.192.206.99", 2562), # Heartbeat/status + ("226.192.206.100", 2563), # Alternative data (low traffic) + ("226.192.206.101", 2564), # Alternative data (low traffic) + ("226.192.206.102", 2565), # PRIMARY sensor data + ("226.192.219.0", 3221), # Display synchronization (HIGH traffic, no sensor data!) + ("239.2.1.1", 2154), # Tank/engine data (not used on this vessel) +] + +# Primary multicast group for most sensor data +PRIMARY_MULTICAST_GROUP = ("226.192.206.102", 2565) + + +def get_tank_name(tank_id: int) -> str: + """Get the display name for a tank ID.""" + if tank_id in TANK_CONFIG: + return TANK_CONFIG[tank_id].name + return f"Tank #{tank_id}" + + +def get_tank_capacity(tank_id: int) -> Optional[float]: + """Get the capacity in gallons for a tank ID.""" + if tank_id in TANK_CONFIG: + return TANK_CONFIG[tank_id].capacity_gallons + return None + + +def get_battery_name(battery_id: int) -> str: + """Get the display name for a battery ID.""" + if battery_id in BATTERY_CONFIG: + return BATTERY_CONFIG[battery_id].name + return f"Battery #{battery_id}" + + +def get_battery_nominal_voltage(battery_id: int) -> float: + """Get the nominal voltage for a battery ID.""" + if battery_id in BATTERY_CONFIG: + return BATTERY_CONFIG[battery_id].nominal_voltage + # Default to 12V for unknown batteries + return 12.0 + + +__all__ = [ + "TankInfo", + "BatteryInfo", + "TANK_CONFIG", + "BATTERY_CONFIG", + "MULTICAST_GROUPS", + "MULTICAST_GROUPS_ALL", + "PRIMARY_MULTICAST_GROUP", + "get_tank_name", + "get_tank_capacity", + "get_battery_name", + "get_battery_nominal_voltage", +] diff --git a/axiom-nmea/raymarine_nmea/venus_dbus/__init__.py b/axiom-nmea/raymarine_nmea/venus_dbus/__init__.py new file mode 100644 index 0000000..24a59d4 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/venus_dbus/__init__.py @@ -0,0 +1,51 @@ +""" +Venus OS D-Bus Publisher Module. + +This module provides D-Bus services for publishing Raymarine sensor data +to Venus OS, making it available to the Victron ecosystem. + +Supported services: +- GPS: Position, speed, course, altitude +- Meteo: Wind direction and speed +- Navigation: Heading, depth, water temperature +- Tank: Tank levels for fuel, water, waste +- Battery: Battery voltage and state of charge + +Example usage: + from raymarine_nmea import SensorData, RaymarineDecoder, MulticastListener + from raymarine_nmea.venus_dbus import VenusPublisher + + # Create sensor data store + sensor_data = SensorData() + decoder = RaymarineDecoder() + + # Start multicast listener + listener = MulticastListener( + decoder=decoder, + sensor_data=sensor_data, + interface_ip="198.18.5.5", + ) + listener.start() + + # Start Venus OS publisher + publisher = VenusPublisher(sensor_data) + publisher.start() # Blocks in GLib main loop +""" + +from .service import VeDbusServiceBase +from .gps import GpsService +from .meteo import MeteoService +from .navigation import NavigationService +from .tank import TankService +from .battery import BatteryService +from .publisher import VenusPublisher + +__all__ = [ + "VeDbusServiceBase", + "GpsService", + "MeteoService", + "NavigationService", + "TankService", + "BatteryService", + "VenusPublisher", +] diff --git a/axiom-nmea/raymarine_nmea/venus_dbus/battery.py b/axiom-nmea/raymarine_nmea/venus_dbus/battery.py new file mode 100644 index 0000000..2edcc8e --- /dev/null +++ b/axiom-nmea/raymarine_nmea/venus_dbus/battery.py @@ -0,0 +1,382 @@ +""" +Battery D-Bus service for Venus OS. + +Publishes battery data to the Venus OS D-Bus using the +com.victronenergy.battery service type. + +Each physical battery requires a separate D-Bus service instance. + +D-Bus paths: + /Dc/0/Voltage - Battery voltage in V DC + /Dc/0/Current - Not available from Raymarine + /Dc/0/Power - Not available from Raymarine + /Dc/0/Temperature - Not available from Raymarine + /Soc - Estimated from voltage for 24V AGM batteries + /Connected - Connection status +""" + +import logging +import time +from typing import Any, Dict, Optional, List, Tuple + +from .service import VeDbusServiceBase +from ..data.store import SensorData +from ..sensors import BATTERY_CONFIG, BatteryInfo, get_battery_name, get_battery_nominal_voltage + +logger = logging.getLogger(__name__) + +# Alert thresholds +LOW_VOLTAGE_ALERT_THRESHOLD = 23.0 # Volts - alert if below this +LOW_VOLTAGE_WARNING_DELAY = 60.0 # Seconds - warning (level 1) after this duration +LOW_VOLTAGE_ALARM_DELAY = 300.0 # Seconds - alarm (level 2) after this duration + +# High voltage alert thresholds +HIGH_VOLTAGE_ALERT_THRESHOLD = 30.2 # Volts - alert if above this (above normal absorption) +HIGH_VOLTAGE_WARNING_DELAY = 60.0 # Seconds - warning (level 1) after this duration +HIGH_VOLTAGE_ALARM_DELAY = 300.0 # Seconds - alarm (level 2) after this duration + +# 24V AGM battery voltage to SOC lookup table +# Based on resting voltage (no load, no charge) for AGM batteries +# Format: (voltage, soc_percent) +# 24V = 2x 12V batteries in series +AGM_24V_SOC_TABLE: List[Tuple[float, int]] = [ + (25.50, 100), # 12.75V per battery - fully charged + (25.30, 95), + (25.10, 90), + (24.90, 85), + (24.70, 80), + (24.50, 75), + (24.30, 70), + (24.10, 65), + (23.90, 60), + (23.70, 55), + (23.50, 50), + (23.30, 45), + (23.10, 40), + (22.90, 35), + (22.70, 30), + (22.50, 25), + (22.30, 20), + (22.10, 15), + (21.90, 10), + (21.70, 5), + (21.50, 0), # 10.75V per battery - fully discharged +] + + +def estimate_soc_24v_agm(voltage: float) -> int: + """Estimate state of charge for a 24V AGM battery based on voltage. + + This is an approximation based on resting voltage. Actual SOC can vary + based on load, temperature, and battery age. + + Args: + voltage: Battery voltage in volts + + Returns: + Estimated SOC as percentage (0-100) + """ + if voltage >= AGM_24V_SOC_TABLE[0][0]: + return 100 + if voltage <= AGM_24V_SOC_TABLE[-1][0]: + return 0 + + # Find the two points to interpolate between + for i in range(len(AGM_24V_SOC_TABLE) - 1): + v_high, soc_high = AGM_24V_SOC_TABLE[i] + v_low, soc_low = AGM_24V_SOC_TABLE[i + 1] + + if v_low <= voltage <= v_high: + # Linear interpolation + ratio = (voltage - v_low) / (v_high - v_low) + soc = soc_low + ratio * (soc_high - soc_low) + return int(round(soc)) + + return 50 # Fallback + + +class BatteryService(VeDbusServiceBase): + """Battery D-Bus service for Venus OS. + + Publishes a single battery's voltage data to the Venus OS D-Bus. + Create one instance per physical battery. + + Note: Raymarine only provides voltage readings. Current, power, + SOC, and other advanced metrics require a dedicated battery monitor + like a Victron BMV or SmartShunt. + + Example: + sensor_data = SensorData() + + # Create service for battery ID 11 + battery_service = BatteryService( + sensor_data=sensor_data, + battery_id=11, + device_instance=0, + ) + battery_service.register() + + # In update loop: + battery_service.update() + """ + + service_type = "battery" + product_name = "Raymarine Battery Monitor" + product_id = 0xA143 # Custom product ID for Raymarine Battery + + # Maximum age in seconds before data is considered stale + MAX_DATA_AGE = 30.0 + + def __init__( + self, + sensor_data: SensorData, + battery_id: int, + device_instance: int = 0, + battery_config: Optional[BatteryInfo] = None, + custom_name: Optional[str] = None, + ): + """Initialize Battery service. + + Args: + sensor_data: SensorData instance to read values from + battery_id: The Raymarine battery ID + device_instance: Unique instance number for this battery + battery_config: Optional BatteryInfo override + custom_name: Optional custom display name + """ + # Get battery configuration + if battery_config: + self._battery_config = battery_config + elif battery_id in BATTERY_CONFIG: + self._battery_config = BATTERY_CONFIG[battery_id] + else: + self._battery_config = BatteryInfo(f"Battery #{battery_id}", 12.0, "house") + + # Use battery name as product name + self.product_name = self._battery_config.name + + super().__init__( + device_instance=device_instance, + connection=f"Raymarine Battery {battery_id}", + custom_name=custom_name or self._battery_config.name, + ) + self._sensor_data = sensor_data + self._battery_id = battery_id + + # Track when voltage first dropped below threshold for delayed alert + self._low_voltage_since: Optional[float] = None + self._low_voltage_warned = False # Tracks if warning (level 1) was logged + self._low_voltage_alerted = False # Tracks if alarm (level 2) was logged + + # Track when voltage first exceeded threshold for delayed alert + self._high_voltage_since: Optional[float] = None + self._high_voltage_warned = False # Tracks if warning (level 1) was logged + self._high_voltage_alerted = False # Tracks if alarm (level 2) was logged + + @property + def service_name(self) -> str: + """Get the full D-Bus service name.""" + return f"com.victronenergy.battery.raymarine_bat{self._battery_id}_{self.device_instance}" + + def _get_paths(self) -> Dict[str, Dict[str, Any]]: + """Return battery-specific D-Bus paths.""" + return { + # DC measurements + '/Dc/0/Voltage': {'initial': None}, + '/Dc/0/Current': {'initial': None}, + '/Dc/0/Power': {'initial': None}, + '/Dc/0/Temperature': {'initial': None}, + + # State of charge (not available from Raymarine) + '/Soc': {'initial': None}, + '/TimeToGo': {'initial': None}, + '/ConsumedAmphours': {'initial': None}, + + # Alarms (not monitored via Raymarine) + '/Alarms/LowVoltage': {'initial': 0}, + '/Alarms/HighVoltage': {'initial': 0}, + '/Alarms/LowSoc': {'initial': None}, + '/Alarms/LowTemperature': {'initial': None}, + '/Alarms/HighTemperature': {'initial': None}, + + # Settings + '/Settings/HasTemperature': {'initial': 0}, + '/Settings/HasStarterVoltage': {'initial': 0}, + '/Settings/HasMidVoltage': {'initial': 0}, + } + + def _update(self) -> None: + """Update battery values from sensor data.""" + data = self._sensor_data + now = time.time() + + # Check data freshness + is_stale = data.is_stale('battery', self.MAX_DATA_AGE) + + # Get battery voltage from live data + voltage = data.batteries.get(self._battery_id) + + if voltage is not None and not is_stale: + # Valid battery data + self._set_value('/Dc/0/Voltage', round(voltage, 2)) + + # Estimate SOC for 24V AGM batteries + nominal = self._battery_config.nominal_voltage + if nominal == 24.0: + soc = estimate_soc_24v_agm(voltage) + self._set_value('/Soc', soc) + else: + self._set_value('/Soc', None) + + # Low voltage alert with hysteresis + # Tiered alerts prevent false alarms from temporary voltage drops (e.g., engine start) + # - 0-60 seconds: No alarm (monitoring) + # - 60-300 seconds: Warning (level 1) + # - 300+ seconds: Alarm (level 2) + if voltage < LOW_VOLTAGE_ALERT_THRESHOLD: + if self._low_voltage_since is None: + # First time below threshold - start tracking but don't alert yet + self._low_voltage_since = now + logger.info( + f"{self._battery_config.name}: Voltage {voltage:.1f}V dropped below " + f"{LOW_VOLTAGE_ALERT_THRESHOLD}V threshold, monitoring..." + ) + + elapsed = now - self._low_voltage_since + if elapsed >= LOW_VOLTAGE_ALARM_DELAY: + # Below threshold for 5+ minutes - ALARM (level 2) + self._set_value('/Alarms/LowVoltage', 2) + if not self._low_voltage_alerted: + logger.error( + f"ALARM: {self._battery_config.name} voltage {voltage:.1f}V " + f"has been below {LOW_VOLTAGE_ALERT_THRESHOLD}V for " + f"{elapsed:.0f} seconds!" + ) + self._low_voltage_alerted = True + elif elapsed >= LOW_VOLTAGE_WARNING_DELAY: + # Below threshold for 1-5 minutes - WARNING (level 1) + self._set_value('/Alarms/LowVoltage', 1) + if not self._low_voltage_warned: + logger.warning( + f"WARNING: {self._battery_config.name} voltage {voltage:.1f}V " + f"has been below {LOW_VOLTAGE_ALERT_THRESHOLD}V for " + f"{elapsed:.0f} seconds" + ) + self._low_voltage_warned = True + else: + # Below threshold but within hysteresis period - no alarm yet + self._set_value('/Alarms/LowVoltage', 0) + else: + # Voltage is OK - reset tracking + if self._low_voltage_since is not None: + logger.info( + f"{self._battery_config.name}: Voltage recovered to {voltage:.1f}V" + ) + self._low_voltage_since = None + self._low_voltage_warned = False + self._low_voltage_alerted = False + self._set_value('/Alarms/LowVoltage', 0) + + # High voltage alert with hysteresis + # Tiered alerts prevent false alarms from temporary voltage spikes (e.g., charging) + # - 0-60 seconds: No alarm (monitoring) + # - 60-300 seconds: Warning (level 1) + # - 300+ seconds: Alarm (level 2) + if voltage > HIGH_VOLTAGE_ALERT_THRESHOLD: + if self._high_voltage_since is None: + # First time above threshold - start tracking but don't alert yet + self._high_voltage_since = now + logger.info( + f"{self._battery_config.name}: Voltage {voltage:.1f}V exceeded " + f"{HIGH_VOLTAGE_ALERT_THRESHOLD}V threshold, monitoring..." + ) + + elapsed = now - self._high_voltage_since + if elapsed >= HIGH_VOLTAGE_ALARM_DELAY: + # Above threshold for 5+ minutes - ALARM (level 2) + self._set_value('/Alarms/HighVoltage', 2) + if not self._high_voltage_alerted: + logger.error( + f"ALARM: {self._battery_config.name} voltage {voltage:.1f}V " + f"has been above {HIGH_VOLTAGE_ALERT_THRESHOLD}V for " + f"{elapsed:.0f} seconds!" + ) + self._high_voltage_alerted = True + elif elapsed >= HIGH_VOLTAGE_WARNING_DELAY: + # Above threshold for 1-5 minutes - WARNING (level 1) + self._set_value('/Alarms/HighVoltage', 1) + if not self._high_voltage_warned: + logger.warning( + f"WARNING: {self._battery_config.name} voltage {voltage:.1f}V " + f"has been above {HIGH_VOLTAGE_ALERT_THRESHOLD}V for " + f"{elapsed:.0f} seconds" + ) + self._high_voltage_warned = True + else: + # Above threshold but within hysteresis period - no alarm yet + self._set_value('/Alarms/HighVoltage', 0) + else: + # Voltage is OK - reset tracking + if self._high_voltage_since is not None: + logger.info( + f"{self._battery_config.name}: Voltage recovered to {voltage:.1f}V" + ) + self._high_voltage_since = None + self._high_voltage_warned = False + self._high_voltage_alerted = False + self._set_value('/Alarms/HighVoltage', 0) + + else: + # No data or stale - show as unavailable + self._set_value('/Dc/0/Voltage', None) + self._set_value('/Soc', None) + self._set_value('/Alarms/LowVoltage', 0) + self._set_value('/Alarms/HighVoltage', 0) + # Reset voltage tracking when data is stale + self._low_voltage_since = None + self._low_voltage_warned = False + self._low_voltage_alerted = False + self._high_voltage_since = None + self._high_voltage_warned = False + self._high_voltage_alerted = False + + # These are not available from Raymarine + self._set_value('/Dc/0/Current', None) + self._set_value('/Dc/0/Power', None) + self._set_value('/Dc/0/Temperature', None) + self._set_value('/TimeToGo', None) + self._set_value('/ConsumedAmphours', None) + + # Update connection status + self.set_connected(voltage is not None and not is_stale) + + +def create_battery_services( + sensor_data: SensorData, + battery_ids: Optional[List[int]] = None, + start_instance: int = 0, +) -> List[BatteryService]: + """Create BatteryService instances for multiple batteries. + + Args: + sensor_data: SensorData instance to read values from + battery_ids: List of battery IDs to create services for. + If None, creates services for all configured batteries. + start_instance: Starting device instance number + + Returns: + List of BatteryService instances + """ + if battery_ids is None: + battery_ids = list(BATTERY_CONFIG.keys()) + + services = [] + for i, battery_id in enumerate(battery_ids): + service = BatteryService( + sensor_data=sensor_data, + battery_id=battery_id, + device_instance=start_instance + i, + ) + services.append(service) + + return services diff --git a/axiom-nmea/raymarine_nmea/venus_dbus/gps.py b/axiom-nmea/raymarine_nmea/venus_dbus/gps.py new file mode 100644 index 0000000..4ecfe01 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/venus_dbus/gps.py @@ -0,0 +1,134 @@ +""" +GPS D-Bus service for Venus OS. + +Publishes GPS position, speed, and course data to the Venus OS D-Bus +using the com.victronenergy.gps service type. + +D-Bus paths: + /Altitude - Height in meters + /Course - Direction in degrees (COG) + /Fix - GPS fix status (0=no fix, 1=fix) + /NrOfSatellites - Number of satellites (not available from Raymarine) + /Position/Latitude - Latitude in degrees + /Position/Longitude - Longitude in degrees + /Speed - Speed in m/s (SOG) +""" + +import logging +from typing import Any, Dict, Optional + +from .service import VeDbusServiceBase +from ..data.store import SensorData +from ..protocol.constants import MS_TO_KTS + +logger = logging.getLogger(__name__) + +# Conversion: knots to m/s +KTS_TO_MS = 1.0 / MS_TO_KTS # Approximately 0.514444 + + +class GpsService(VeDbusServiceBase): + """GPS D-Bus service for Venus OS. + + Publishes GPS position, speed, and course from Raymarine sensors + to the Venus OS D-Bus. + + Example: + sensor_data = SensorData() + gps_service = GpsService(sensor_data) + gps_service.register() + + # In update loop: + gps_service.update() + """ + + service_type = "gps" + product_name = "Raymarine GPS" + product_id = 0xA140 # Custom product ID for Raymarine GPS + + # Maximum age in seconds before GPS data is considered stale + MAX_DATA_AGE = 10.0 + + def __init__( + self, + sensor_data: SensorData, + device_instance: int = 0, + custom_name: Optional[str] = None, + ): + """Initialize GPS service. + + Args: + sensor_data: SensorData instance to read GPS values from + device_instance: Unique instance number (default: 0) + custom_name: Optional custom display name + """ + super().__init__( + device_instance=device_instance, + connection="Raymarine LightHouse GPS", + custom_name=custom_name, + ) + self._sensor_data = sensor_data + + def _get_paths(self) -> Dict[str, Dict[str, Any]]: + """Return GPS-specific D-Bus paths.""" + return { + '/Altitude': {'initial': None}, + '/Course': {'initial': None}, + '/Fix': {'initial': 0}, + '/NrOfSatellites': {'initial': None}, + '/Position/Latitude': {'initial': None}, + '/Position/Longitude': {'initial': None}, + '/Speed': {'initial': None}, + } + + def _update(self) -> None: + """Update GPS values from sensor data.""" + data = self._sensor_data + + # Check if we have valid GPS data + has_position = ( + data.latitude is not None and + data.longitude is not None + ) + + # Check data freshness + is_stale = data.is_stale('gps', self.MAX_DATA_AGE) + + if has_position and not is_stale: + # Valid GPS fix + self._set_value('/Fix', 1) + self._set_value('/Position/Latitude', data.latitude) + self._set_value('/Position/Longitude', data.longitude) + + # Course over ground (degrees) + if data.cog_deg is not None: + self._set_value('/Course', round(data.cog_deg, 1)) + else: + self._set_value('/Course', None) + + # Speed over ground (convert knots to m/s) + if data.sog_kts is not None: + speed_ms = data.sog_kts * KTS_TO_MS + self._set_value('/Speed', round(speed_ms, 2)) + else: + self._set_value('/Speed', None) + + # Altitude not available from Raymarine multicast + # (would need NMEA GGA sentence with altitude field) + self._set_value('/Altitude', None) + + # Number of satellites not available from Raymarine + self._set_value('/NrOfSatellites', None) + + else: + # No GPS fix or stale data + self._set_value('/Fix', 0) + self._set_value('/Position/Latitude', None) + self._set_value('/Position/Longitude', None) + self._set_value('/Course', None) + self._set_value('/Speed', None) + self._set_value('/Altitude', None) + self._set_value('/NrOfSatellites', None) + + # Update connection status + self.set_connected(not is_stale and has_position) diff --git a/axiom-nmea/raymarine_nmea/venus_dbus/meteo.py b/axiom-nmea/raymarine_nmea/venus_dbus/meteo.py new file mode 100644 index 0000000..aad13c0 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/venus_dbus/meteo.py @@ -0,0 +1,151 @@ +""" +Meteo (Weather) D-Bus service for Venus OS. + +Publishes wind and environmental data to the Venus OS D-Bus +using the com.victronenergy.meteo service type. + +D-Bus paths: + /WindDirection - True wind direction in degrees (0-360) + /WindSpeed - True wind speed in m/s (Venus OS requirement) + /ExternalTemperature - Air temperature in Celsius + /CellTemperature - Not used (panel temperature for solar) + /Irradiance - Not used (solar irradiance) +""" + +import logging +from typing import Any, Dict, Optional + +from .service import VeDbusServiceBase +from ..data.store import SensorData +from ..protocol.constants import MS_TO_KTS + +logger = logging.getLogger(__name__) + +# Conversion: knots to m/s +KTS_TO_MS = 1.0 / MS_TO_KTS # Approximately 0.514444 + + +class MeteoService(VeDbusServiceBase): + """Meteo (Weather) D-Bus service for Venus OS. + + Publishes wind direction, wind speed, and temperature from + Raymarine sensors to the Venus OS D-Bus. + + Example: + sensor_data = SensorData() + meteo_service = MeteoService(sensor_data) + meteo_service.register() + + # In update loop: + meteo_service.update() + """ + + service_type = "meteo" + product_name = "Weather" + product_id = 0xA141 # Custom product ID for Raymarine Meteo + + # Maximum age in seconds before data is considered stale + MAX_DATA_AGE = 10.0 + + def __init__( + self, + sensor_data: SensorData, + device_instance: int = 0, + custom_name: Optional[str] = None, + ): + """Initialize Meteo service. + + Args: + sensor_data: SensorData instance to read values from + device_instance: Unique instance number (default: 0) + custom_name: Optional custom display name + """ + super().__init__( + device_instance=device_instance, + connection="Raymarine LightHouse Weather", + custom_name=custom_name, + ) + self._sensor_data = sensor_data + + def _get_paths(self) -> Dict[str, Dict[str, Any]]: + """Return meteo-specific D-Bus paths.""" + return { + # Standard meteo paths + '/WindDirection': {'initial': None}, + '/WindSpeed': {'initial': None}, + '/ExternalTemperature': {'initial': None}, + '/CellTemperature': {'initial': None}, + '/Irradiance': {'initial': None}, + '/ErrorCode': {'initial': 0}, + + # Extended paths for apparent wind + # These may not be recognized by all Venus OS components + # but are useful for custom dashboards + '/ApparentWindAngle': {'initial': None}, + '/ApparentWindSpeed': {'initial': None}, + + # Barometric pressure (custom extension) + '/Pressure': {'initial': None}, + } + + def _update(self) -> None: + """Update meteo values from sensor data.""" + data = self._sensor_data + + # Check data freshness + wind_stale = data.is_stale('wind', self.MAX_DATA_AGE) + temp_stale = data.is_stale('temp', self.MAX_DATA_AGE) + pressure_stale = data.is_stale('pressure', self.MAX_DATA_AGE) + + # True wind direction (degrees 0-360) + if not wind_stale and data.twd_deg is not None: + self._set_value('/WindDirection', round(data.twd_deg, 1)) + else: + self._set_value('/WindDirection', None) + + # True wind speed (convert knots to m/s for Venus OS) + if not wind_stale and data.tws_kts is not None: + speed_ms = data.tws_kts * KTS_TO_MS + self._set_value('/WindSpeed', round(speed_ms, 2)) + else: + self._set_value('/WindSpeed', None) + + # Apparent wind angle (degrees, relative to bow) + if not wind_stale and data.awa_deg is not None: + self._set_value('/ApparentWindAngle', round(data.awa_deg, 1)) + else: + self._set_value('/ApparentWindAngle', None) + + # Apparent wind speed (convert knots to m/s for Venus OS) + if not wind_stale and data.aws_kts is not None: + speed_ms = data.aws_kts * KTS_TO_MS + self._set_value('/ApparentWindSpeed', round(speed_ms, 2)) + else: + self._set_value('/ApparentWindSpeed', None) + + # Air temperature (Celsius) + if not temp_stale and data.air_temp_c is not None: + self._set_value('/ExternalTemperature', round(data.air_temp_c, 1)) + else: + self._set_value('/ExternalTemperature', None) + + # Barometric pressure (convert mbar to hPa - they're equivalent) + if not pressure_stale and data.pressure_mbar is not None: + self._set_value('/Pressure', round(data.pressure_mbar, 1)) + else: + self._set_value('/Pressure', None) + + # Cell temperature and irradiance are not available + self._set_value('/CellTemperature', None) + self._set_value('/Irradiance', None) + + # Error code: 0 = OK + has_any_data = ( + (not wind_stale and data.twd_deg is not None) or + (not temp_stale and data.air_temp_c is not None) or + (not pressure_stale and data.pressure_mbar is not None) + ) + self._set_value('/ErrorCode', 0 if has_any_data else 1) + + # Update connection status + self.set_connected(not wind_stale) diff --git a/axiom-nmea/raymarine_nmea/venus_dbus/navigation.py b/axiom-nmea/raymarine_nmea/venus_dbus/navigation.py new file mode 100644 index 0000000..bca7f9c --- /dev/null +++ b/axiom-nmea/raymarine_nmea/venus_dbus/navigation.py @@ -0,0 +1,102 @@ +""" +Navigation D-Bus service for Venus OS. + +Publishes navigation data not covered by the standard GPS or Meteo +services to the Venus OS D-Bus, making it available to custom addons +via D-Bus and MQTT. + +D-Bus paths: + /Heading - True heading in degrees (0-360) + /Depth - Depth below transducer in meters + /WaterTemperature - Water temperature in Celsius +""" + +import logging +from typing import Any, Dict, Optional + +from .service import VeDbusServiceBase +from ..data.store import SensorData + +logger = logging.getLogger(__name__) + + +class NavigationService(VeDbusServiceBase): + """Navigation D-Bus service for Venus OS. + + Publishes heading, depth, and water temperature from Raymarine + sensors to the Venus OS D-Bus. These values are decoded from the + Raymarine protocol but don't map to standard Venus OS service types, + so they are grouped under a custom 'navigation' service. + + Example: + sensor_data = SensorData() + nav_service = NavigationService(sensor_data) + nav_service.register() + + # In update loop: + nav_service.update() + """ + + service_type = "navigation" + product_name = "Raymarine Navigation" + product_id = 0xA143 + + MAX_DATA_AGE = 10.0 + + def __init__( + self, + sensor_data: SensorData, + device_instance: int = 0, + custom_name: Optional[str] = None, + ): + """Initialize Navigation service. + + Args: + sensor_data: SensorData instance to read values from + device_instance: Unique instance number (default: 0) + custom_name: Optional custom display name + """ + super().__init__( + device_instance=device_instance, + connection="Raymarine LightHouse Navigation", + custom_name=custom_name, + ) + self._sensor_data = sensor_data + + def _get_paths(self) -> Dict[str, Dict[str, Any]]: + """Return navigation-specific D-Bus paths.""" + return { + '/Heading': {'initial': None}, + '/Depth': {'initial': None}, + '/WaterTemperature': {'initial': None}, + } + + def _update(self) -> None: + """Update navigation values from sensor data.""" + data = self._sensor_data + + heading_stale = data.is_stale('heading', self.MAX_DATA_AGE) + depth_stale = data.is_stale('depth', self.MAX_DATA_AGE) + temp_stale = data.is_stale('temp', self.MAX_DATA_AGE) + + if not heading_stale and data.heading_deg is not None: + self._set_value('/Heading', round(data.heading_deg, 1)) + else: + self._set_value('/Heading', None) + + if not depth_stale and data.depth_m is not None: + self._set_value('/Depth', round(data.depth_m, 2)) + else: + self._set_value('/Depth', None) + + if not temp_stale and data.water_temp_c is not None: + self._set_value('/WaterTemperature', round(data.water_temp_c, 1)) + else: + self._set_value('/WaterTemperature', None) + + has_any_data = ( + (not heading_stale and data.heading_deg is not None) or + (not depth_stale and data.depth_m is not None) or + (not temp_stale and data.water_temp_c is not None) + ) + self.set_connected(has_any_data) diff --git a/axiom-nmea/raymarine_nmea/venus_dbus/publisher.py b/axiom-nmea/raymarine_nmea/venus_dbus/publisher.py new file mode 100644 index 0000000..aee99bf --- /dev/null +++ b/axiom-nmea/raymarine_nmea/venus_dbus/publisher.py @@ -0,0 +1,290 @@ +""" +Venus OS D-Bus Publisher. + +This module provides the main VenusPublisher class that coordinates +all D-Bus services for publishing Raymarine sensor data to Venus OS. +""" + +import logging +import signal +import sys +from typing import List, Optional, Set + +from ..data.store import SensorData +from ..sensors import TANK_CONFIG, BATTERY_CONFIG +from .gps import GpsService +from .meteo import MeteoService +from .navigation import NavigationService +from .tank import TankService, create_tank_services +from .battery import BatteryService, create_battery_services +from .service import VeDbusServiceBase + +logger = logging.getLogger(__name__) + +# Try to import GLib for the main loop +try: + from gi.repository import GLib + HAS_GLIB = True +except ImportError: + HAS_GLIB = False + logger.warning("GLib not available. VenusPublisher.run() will not work.") + + +class VenusPublisher: + """Coordinator for all Venus OS D-Bus services. + + This class manages the lifecycle of GPS, Meteo, Navigation, Tank, + and Battery D-Bus services, handling registration, updates, and cleanup. + + Example: + from raymarine_nmea import SensorData, RaymarineDecoder, MulticastListener + from raymarine_nmea.venus_dbus import VenusPublisher + + # Create sensor data store + sensor_data = SensorData() + decoder = RaymarineDecoder() + + # Start multicast listener + listener = MulticastListener( + decoder=decoder, + sensor_data=sensor_data, + interface_ip="198.18.5.5", + ) + listener.start() + + # Start Venus OS publisher + publisher = VenusPublisher(sensor_data) + publisher.run() # Blocks until stopped + + For more control over the main loop: + publisher = VenusPublisher(sensor_data) + publisher.start() # Non-blocking, registers services + + # Your own main loop here + # Call publisher.update() periodically + + publisher.stop() # Cleanup + """ + + # Default update interval in milliseconds + DEFAULT_UPDATE_INTERVAL_MS = 1000 + + def __init__( + self, + sensor_data: SensorData, + enable_gps: bool = True, + enable_meteo: bool = True, + enable_navigation: bool = True, + enable_tanks: bool = True, + enable_batteries: bool = True, + tank_ids: Optional[List[int]] = None, + battery_ids: Optional[List[int]] = None, + update_interval_ms: int = DEFAULT_UPDATE_INTERVAL_MS, + ): + """Initialize Venus Publisher. + + Args: + sensor_data: SensorData instance to read values from + enable_gps: Enable GPS service (default: True) + enable_meteo: Enable Meteo/wind service (default: True) + enable_navigation: Enable Navigation service (default: True) + enable_tanks: Enable Tank services (default: True) + enable_batteries: Enable Battery services (default: True) + tank_ids: Specific tank IDs to publish (default: all configured) + battery_ids: Specific battery IDs to publish (default: all configured) + update_interval_ms: Update interval in milliseconds (default: 1000) + """ + self._sensor_data = sensor_data + self._update_interval_ms = update_interval_ms + self._services: List[VeDbusServiceBase] = [] + self._running = False + self._mainloop = None + self._timer_id = None + + # Create enabled services + if enable_gps: + self._services.append(GpsService(sensor_data)) + + if enable_meteo: + self._services.append(MeteoService(sensor_data)) + + if enable_navigation: + self._services.append(NavigationService(sensor_data)) + + if enable_tanks: + self._services.extend( + create_tank_services(sensor_data, tank_ids) + ) + + if enable_batteries: + self._services.extend( + create_battery_services(sensor_data, battery_ids) + ) + + logger.info(f"VenusPublisher initialized with {len(self._services)} services") + + def start(self) -> bool: + """Register all D-Bus services. + + Returns: + True if at least one service registered successfully + """ + if self._running: + logger.warning("VenusPublisher already running") + return True + + registered = 0 + for service in self._services: + if service.register(): + registered += 1 + else: + logger.warning(f"Failed to register {service.service_name}") + + self._running = registered > 0 + + if self._running: + logger.info(f"VenusPublisher started: {registered}/{len(self._services)} services registered") + else: + logger.error("VenusPublisher failed to start: no services registered") + + return self._running + + def stop(self) -> None: + """Stop and unregister all D-Bus services.""" + if not self._running: + return + + # Stop timer if running in GLib main loop + if self._timer_id is not None and HAS_GLIB: + GLib.source_remove(self._timer_id) + self._timer_id = None + + # Quit main loop if running + if self._mainloop is not None: + self._mainloop.quit() + self._mainloop = None + + # Unregister all services + for service in self._services: + service.unregister() + + self._running = False + logger.info("VenusPublisher stopped") + + def update(self) -> bool: + """Update all D-Bus services. + + Call this periodically to refresh values. + + Returns: + True to continue updates, False if all services failed + """ + if not self._running: + return False + + success = 0 + for service in self._services: + if service.update(): + success += 1 + + return success > 0 + + def run(self) -> None: + """Run the publisher with a GLib main loop. + + This method blocks until the publisher is stopped via stop() + or a SIGINT/SIGTERM signal is received. + + Raises: + RuntimeError: If GLib is not available + """ + if not HAS_GLIB: + raise RuntimeError( + "GLib is required to run VenusPublisher. " + "Install PyGObject or use start()/update()/stop() manually." + ) + + # Set up D-Bus main loop + try: + from dbus.mainloop.glib import DBusGMainLoop + DBusGMainLoop(set_as_default=True) + except ImportError: + raise RuntimeError( + "dbus-python with GLib support is required. " + "Install python3-dbus on Venus OS." + ) + + # Start services + if not self.start(): + logger.error("Failed to start VenusPublisher") + return + + # Set up signal handlers for graceful shutdown + def signal_handler(signum, frame): + logger.info(f"Received signal {signum}, stopping...") + self.stop() + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # Set up periodic updates + def update_callback(): + if not self._running: + return False + return self.update() + + self._timer_id = GLib.timeout_add( + self._update_interval_ms, + update_callback + ) + + # Run main loop + logger.info("VenusPublisher running, press Ctrl+C to stop") + self._mainloop = GLib.MainLoop() + + try: + self._mainloop.run() + except Exception as e: + logger.error(f"Main loop error: {e}") + finally: + self.stop() + + @property + def services(self) -> List[VeDbusServiceBase]: + """Get list of all managed services.""" + return self._services.copy() + + @property + def is_running(self) -> bool: + """Check if publisher is running.""" + return self._running + + def add_service(self, service: VeDbusServiceBase) -> bool: + """Add a custom service to the publisher. + + Args: + service: Service instance to add + + Returns: + True if added (and registered if already running) + """ + self._services.append(service) + + if self._running: + return service.register() + return True + + def get_service_status(self) -> dict: + """Get status of all services. + + Returns: + Dict with service names and their registration status + """ + return { + service.service_name: { + 'registered': service._registered, + 'type': service.service_type, + 'product': service.product_name, + } + for service in self._services + } diff --git a/axiom-nmea/raymarine_nmea/venus_dbus/service.py b/axiom-nmea/raymarine_nmea/venus_dbus/service.py new file mode 100644 index 0000000..6c49920 --- /dev/null +++ b/axiom-nmea/raymarine_nmea/venus_dbus/service.py @@ -0,0 +1,301 @@ +""" +Base D-Bus service class for Venus OS integration. + +This module provides a base class that wraps the VeDbusService from +Victron's velib_python library, following their standard patterns. +""" + +import logging +import platform +import os +import sys +from abc import ABC, abstractmethod +from typing import Any, Dict, Optional, Callable + +logger = logging.getLogger(__name__) + +# Venus OS stores velib_python in /opt/victronenergy +VELIB_PATHS = [ + '/opt/victronenergy/dbus-systemcalc-py/ext/velib_python', + '/opt/victronenergy/velib_python', + os.path.join(os.path.dirname(__file__), '../../ext/velib_python'), +] + +# Try to import vedbus from velib_python +VeDbusService = None +dbusconnection = None +for path in VELIB_PATHS: + if os.path.exists(path): + if path not in sys.path: + sys.path.insert(0, path) + try: + from vedbus import VeDbusService + # Also import dbusconnection for creating separate connections + try: + from dbusmonitor import DbusMonitor + import dbus + except ImportError: + pass + logger.debug(f"Loaded VeDbusService from {path}") + break + except ImportError: + if path in sys.path: + sys.path.remove(path) + continue + +if VeDbusService is None: + logger.warning( + "VeDbusService not available. Venus OS D-Bus publishing will be disabled. " + "This is expected when not running on Venus OS." + ) + + +def get_dbus_connection(): + """Get a new private D-Bus connection. + + Each service needs its own connection to avoid path conflicts. + """ + try: + import dbus + return dbus.SystemBus(private=True) + except Exception as e: + logger.error(f"Failed to create D-Bus connection: {e}") + return None + + +class VeDbusServiceBase(ABC): + """Base class for Venus OS D-Bus services. + + This class provides common functionality for creating D-Bus services + that follow Victron's standards for Venus OS integration. + + Subclasses must implement: + - service_type: The service type (e.g., 'gps', 'tank', 'battery') + - product_name: Human-readable product name + - _get_paths(): Returns dict of D-Bus paths to register + - _update(): Called periodically to update values + + Example: + class MyService(VeDbusServiceBase): + service_type = 'myservice' + product_name = 'My Custom Service' + + def _get_paths(self): + return { + '/Value': {'initial': 0}, + '/Status': {'initial': 'OK'}, + } + + def _update(self): + self._set_value('/Value', 42) + """ + + # Subclasses must define these + service_type: str = "" + product_name: str = "Raymarine Sensor" + product_id: int = 0xFFFF # Custom product ID + + def __init__( + self, + device_instance: int = 0, + connection: str = "Raymarine LightHouse", + custom_name: Optional[str] = None, + ): + """Initialize the D-Bus service. + + Args: + device_instance: Unique instance number for this device type + connection: Connection description string + custom_name: Optional custom name override + """ + self.device_instance = device_instance + self.connection = connection + self.custom_name = custom_name + self._dbusservice = None + self._bus = None # Private D-Bus connection for this service + self._paths: Dict[str, Any] = {} + self._registered = False + + @property + def service_name(self) -> str: + """Get the full D-Bus service name.""" + return f"com.victronenergy.{self.service_type}.raymarine_{self.device_instance}" + + @abstractmethod + def _get_paths(self) -> Dict[str, Dict[str, Any]]: + """Return the D-Bus paths to register. + + Returns: + Dict mapping path names to their settings. + Each setting dict can contain: + - initial: Initial value + - writeable: Whether external writes are allowed (default: False) + """ + pass + + @abstractmethod + def _update(self) -> None: + """Update the D-Bus values. + + Called periodically to refresh values from sensor data. + Use _set_value() to update individual paths. + """ + pass + + def register(self) -> bool: + """Register the service with D-Bus. + + Returns: + True if registration succeeded, False otherwise + """ + if VeDbusService is None: + logger.warning(f"Cannot register {self.service_name}: VeDbusService not available") + return False + + if self._registered: + logger.warning(f"Service {self.service_name} already registered") + return True + + try: + # Create a private D-Bus connection for this service + # Each service needs its own connection to avoid path conflicts + self._bus = get_dbus_connection() + if self._bus is None: + logger.error(f"Failed to get D-Bus connection for {self.service_name}") + return False + + # Create service with register=False (new API requirement) + # This allows us to add all paths before registering + self._dbusservice = VeDbusService( + self.service_name, + bus=self._bus, + register=False + ) + + # Create management objects + self._dbusservice.add_path('/Mgmt/ProcessName', __file__) + self._dbusservice.add_path( + '/Mgmt/ProcessVersion', + f'1.0 (Python {platform.python_version()})' + ) + self._dbusservice.add_path('/Mgmt/Connection', self.connection) + + # Create mandatory objects + self._dbusservice.add_path('/DeviceInstance', self.device_instance) + self._dbusservice.add_path('/ProductId', self.product_id) + self._dbusservice.add_path( + '/ProductName', + self.custom_name or self.product_name + ) + self._dbusservice.add_path('/FirmwareVersion', 1) + self._dbusservice.add_path('/HardwareVersion', 0) + self._dbusservice.add_path('/Connected', 1) + + # Add custom name if supported + if self.custom_name: + self._dbusservice.add_path('/CustomName', self.custom_name) + + # Register service-specific paths + self._paths = self._get_paths() + for path, settings in self._paths.items(): + initial = settings.get('initial', None) + writeable = settings.get('writeable', False) + self._dbusservice.add_path( + path, + initial, + writeable=writeable, + onchangecallback=self._handle_changed_value if writeable else None + ) + + # Complete registration after all paths are added + self._dbusservice.register() + self._registered = True + + logger.info(f"Registered D-Bus service: {self.service_name}") + return True + + except Exception as e: + logger.error(f"Failed to register {self.service_name}: {e}") + return False + + def unregister(self) -> None: + """Unregister the service from D-Bus.""" + if self._dbusservice and self._registered: + # VeDbusService doesn't have a clean unregister method, + # so we just mark ourselves as unregistered + self._registered = False + self._dbusservice = None + # Close the private D-Bus connection + if self._bus: + try: + self._bus.close() + except Exception: + pass + self._bus = None + logger.info(f"Unregistered D-Bus service: {self.service_name}") + + def update(self) -> bool: + """Update the service values. + + Returns: + True to continue updates, False to stop + """ + if not self._registered: + return False + + try: + self._update() + return True + except Exception as e: + logger.error(f"Error updating {self.service_name}: {e}") + return True # Keep trying + + def _set_value(self, path: str, value: Any) -> None: + """Set a D-Bus path value. + + Args: + path: The D-Bus path (e.g., '/Position/Latitude') + value: The value to set + """ + if self._dbusservice and self._registered: + with self._dbusservice as s: + if path in s: + s[path] = value + + def _get_value(self, path: str) -> Any: + """Get a D-Bus path value. + + Args: + path: The D-Bus path + + Returns: + The current value, or None if not found + """ + if self._dbusservice and self._registered: + with self._dbusservice as s: + if path in s: + return s[path] + return None + + def _handle_changed_value(self, path: str, value: Any) -> bool: + """Handle external value changes. + + Override this method to handle writes from other processes. + + Args: + path: The D-Bus path that was changed + value: The new value + + Returns: + True to accept the change, False to reject + """ + logger.debug(f"External change: {path} = {value}") + return True + + def set_connected(self, connected: bool) -> None: + """Set the connection status. + + Args: + connected: True if data source is connected + """ + self._set_value('/Connected', 1 if connected else 0) diff --git a/axiom-nmea/raymarine_nmea/venus_dbus/tank.py b/axiom-nmea/raymarine_nmea/venus_dbus/tank.py new file mode 100644 index 0000000..f373dbc --- /dev/null +++ b/axiom-nmea/raymarine_nmea/venus_dbus/tank.py @@ -0,0 +1,349 @@ +""" +Tank D-Bus service for Venus OS. + +Publishes tank level data to the Venus OS D-Bus using the +com.victronenergy.tank service type. + +Each physical tank requires a separate D-Bus service instance. + +D-Bus paths: + /Level - Tank level 0-100% + /Remaining - Remaining volume in m3 + /Status - 0=Ok; 1=Disconnected; 2=Short circuited; 3=Reverse polarity; 4=Unknown + /Capacity - Tank capacity in m3 + /FluidType - 0=Fuel; 1=Fresh water; 2=Waste water; etc. + /Standard - 2 (Not applicable for voltage/current sensors) + + Water tanks (tank_type "water") also publish: + /Alarms/LowLevel - 0=OK, 2=Alarm (level < 10% for 60s) + /Alarms/LowLevelAck - Writable. 0=none, 1=acknowledged, 2=snoozed + /Alarms/LowLevelSnoozeUntil - Writable. Unix timestamp when snooze expires + + Black water tanks (tank_type "blackwater") also publish: + /Alarms/HighLevel - 0=OK, 2=Alarm (level > 75% for 60s) + /Alarms/HighLevelAck - Writable. 0=none, 1=acknowledged, 2=snoozed + /Alarms/HighLevelSnoozeUntil - Writable. Unix timestamp when snooze expires +""" + +import logging +import time +from typing import Any, Dict, Optional, List, Tuple + +from .service import VeDbusServiceBase +from ..data.store import SensorData +from ..sensors import TANK_CONFIG, TankInfo, get_tank_name, get_tank_capacity + +logger = logging.getLogger(__name__) + +# Alarm thresholds +WATER_LOW_LEVEL_THRESHOLD = 10.0 # Alarm if water tank below this percentage +BLACK_WATER_HIGH_LEVEL_THRESHOLD = 75.0 # Alarm if black water above this percentage +TANK_ALARM_DELAY = 60.0 # Seconds before alarm triggers (avoids brief dips) + +# Fuel tank IDs - these get cached when engines are off +FUEL_TANK_IDS = {1, 2} # Fuel Starboard and Fuel Port + +# Memory cache for fuel tank levels (persists while service runs) +# Key: tank_id, Value: (level_percent, timestamp) +_fuel_tank_cache: Dict[int, Tuple[float, float]] = {} + +# Fluid type mapping from tank_type string to Victron enum +FLUID_TYPE_MAP = { + 'fuel': 0, # Fuel + 'water': 1, # Fresh water + 'waste': 2, # Waste water + 'livewell': 3, # Live well + 'oil': 4, # Oil + 'blackwater': 5, # Black water (sewage) + 'gasoline': 6, # Gasoline + 'diesel': 7, # Diesel + 'lpg': 8, # Liquid Petroleum Gas + 'lng': 9, # Liquid Natural Gas + 'hydraulic': 10, # Hydraulic oil + 'rawwater': 11, # Raw water +} + +# Gallons to cubic meters +GALLONS_TO_M3 = 0.00378541 + + +class TankService(VeDbusServiceBase): + """Tank D-Bus service for Venus OS. + + Publishes a single tank's level data to the Venus OS D-Bus. + Create one instance per physical tank. + + Example: + sensor_data = SensorData() + + # Create service for tank ID 1 + tank_service = TankService( + sensor_data=sensor_data, + tank_id=1, + device_instance=0, + ) + tank_service.register() + + # In update loop: + tank_service.update() + """ + + service_type = "tank" + product_name = "Raymarine Tank Sensor" + product_id = 0xA142 # Custom product ID for Raymarine Tank + + # Maximum age in seconds before data is considered stale + MAX_DATA_AGE = 30.0 + + def __init__( + self, + sensor_data: SensorData, + tank_id: int, + device_instance: int = 0, + tank_config: Optional[TankInfo] = None, + custom_name: Optional[str] = None, + ): + """Initialize Tank service. + + Args: + sensor_data: SensorData instance to read values from + tank_id: The Raymarine tank ID + device_instance: Unique instance number for this tank + tank_config: Optional TankInfo override (otherwise uses TANK_CONFIG) + custom_name: Optional custom display name + """ + # Get tank configuration + if tank_config: + self._tank_config = tank_config + elif tank_id in TANK_CONFIG: + self._tank_config = TANK_CONFIG[tank_id] + else: + self._tank_config = TankInfo(f"Tank #{tank_id}", None, "fuel") + + # Use tank name as product name + self.product_name = self._tank_config.name + + super().__init__( + device_instance=device_instance, + connection=f"Raymarine Tank {tank_id}", + custom_name=custom_name or self._tank_config.name, + ) + self._sensor_data = sensor_data + self._tank_id = tank_id + + # Check if this is a fuel tank (should use caching) + self._is_fuel_tank = tank_id in FUEL_TANK_IDS + + # Alarm tracking -- only for water and blackwater tanks + tank_type = self._tank_config.tank_type.lower() + self._has_low_alarm = tank_type == 'water' + self._has_high_alarm = tank_type == 'blackwater' + self._alarm_since: Optional[float] = None + self._alarm_logged = False + + @property + def service_name(self) -> str: + """Get the full D-Bus service name.""" + return f"com.victronenergy.tank.raymarine_tank{self._tank_id}_{self.device_instance}" + + def _get_fluid_type(self) -> int: + """Get the Victron fluid type enum from tank config.""" + tank_type = self._tank_config.tank_type.lower() + return FLUID_TYPE_MAP.get(tank_type, 0) # Default to fuel + + def _get_capacity_m3(self) -> Optional[float]: + """Get tank capacity in cubic meters.""" + if self._tank_config.capacity_gallons is None: + return None + return self._tank_config.capacity_gallons * GALLONS_TO_M3 + + def _get_paths(self) -> Dict[str, Dict[str, Any]]: + """Return tank-specific D-Bus paths.""" + capacity_m3 = self._get_capacity_m3() + + paths: Dict[str, Dict[str, Any]] = { + '/Level': {'initial': None}, + '/Remaining': {'initial': None}, + '/Status': {'initial': 4}, # Unknown until data received + '/Capacity': {'initial': capacity_m3}, + '/FluidType': {'initial': self._get_fluid_type()}, + '/Standard': {'initial': 2}, # Not applicable + } + + if self._has_low_alarm: + paths['/Alarms/LowLevel'] = {'initial': 0} + paths['/Alarms/LowLevelAck'] = {'initial': 0, 'writeable': True} + paths['/Alarms/LowLevelSnoozeUntil'] = {'initial': 0, 'writeable': True} + + if self._has_high_alarm: + paths['/Alarms/HighLevel'] = {'initial': 0} + paths['/Alarms/HighLevelAck'] = {'initial': 0, 'writeable': True} + paths['/Alarms/HighLevelSnoozeUntil'] = {'initial': 0, 'writeable': True} + + return paths + + def _update(self) -> None: + """Update tank values from sensor data.""" + global _fuel_tank_cache + + data = self._sensor_data + now = time.time() + + # Check data freshness + is_stale = data.is_stale('tank', self.MAX_DATA_AGE) + + # Get tank level from live data + level = data.tanks.get(self._tank_id) + using_cached = False + + # For fuel tanks: cache when present, use cache when absent + # Fuel doesn't change when engines are off + if self._is_fuel_tank: + if level is not None and not is_stale: + # Live data available - cache it + _fuel_tank_cache[self._tank_id] = (level, now) + elif self._tank_id in _fuel_tank_cache: + # No live data - use cached value + cached_level, cached_time = _fuel_tank_cache[self._tank_id] + level = cached_level + using_cached = True + # Don't consider cached data as stale for fuel tanks + is_stale = False + + if level is not None and not is_stale: + # Valid tank data (live or cached) + self._set_value('/Level', round(level, 1)) + self._set_value('/Status', 0) # OK + + # Calculate remaining volume + capacity_m3 = self._get_capacity_m3() + if capacity_m3 is not None: + remaining = capacity_m3 * (level / 100.0) + self._set_value('/Remaining', round(remaining, 4)) + else: + self._set_value('/Remaining', None) + + # Check tank-level alarms (skip for cached values) + if not using_cached: + self._check_level_alarm(level, now) + + else: + # No data or stale + self._set_value('/Level', None) + self._set_value('/Remaining', None) + + if is_stale and self._tank_id in data.tanks: + # Had data but now stale + self._set_value('/Status', 1) # Disconnected + else: + # Never had data + self._set_value('/Status', 4) # Unknown + + # Update connection status (cached values count as connected for fuel tanks) + self.set_connected(level is not None and not is_stale) + + def _check_level_alarm(self, level: float, now: float) -> None: + """Check tank level against alarm thresholds and manage alarm state. + + For water tanks: alarms when level < WATER_LOW_LEVEL_THRESHOLD. + For black water: alarms when level > BLACK_WATER_HIGH_LEVEL_THRESHOLD. + Both use TANK_ALARM_DELAY before triggering. + + Also handles snooze expiry: if snoozed (Ack=2) and the snooze time + has passed, resets Ack to 0 so the UI re-alerts. + """ + if self._has_low_alarm: + alarm_path = '/Alarms/LowLevel' + ack_path = '/Alarms/LowLevelAck' + snooze_path = '/Alarms/LowLevelSnoozeUntil' + condition_met = level < WATER_LOW_LEVEL_THRESHOLD + threshold_desc = f"below {WATER_LOW_LEVEL_THRESHOLD}%" + elif self._has_high_alarm: + alarm_path = '/Alarms/HighLevel' + ack_path = '/Alarms/HighLevelAck' + snooze_path = '/Alarms/HighLevelSnoozeUntil' + condition_met = level > BLACK_WATER_HIGH_LEVEL_THRESHOLD + threshold_desc = f"above {BLACK_WATER_HIGH_LEVEL_THRESHOLD}%" + else: + return + + if condition_met: + if self._alarm_since is None: + self._alarm_since = now + + elapsed = now - self._alarm_since + + if elapsed >= TANK_ALARM_DELAY: + self._set_value(alarm_path, 2) + if not self._alarm_logged: + logger.warning( + f"ALARM: {self._tank_config.name} at {level:.1f}% - " + f"{threshold_desc} for {elapsed:.0f}s" + ) + self._alarm_logged = True + + # Check snooze expiry: if snoozed and past expiry, re-trigger + ack_val = self._get_value(ack_path) + snooze_until = self._get_value(snooze_path) + if ack_val == 2 and snooze_until and now > snooze_until: + logger.info( + f"Snooze expired for {self._tank_config.name}, " + f"re-triggering alarm" + ) + self._set_value(ack_path, 0) + self._set_value(snooze_path, 0) + else: + # Condition cleared + if self._alarm_since is not None: + if self._alarm_logged: + logger.info( + f"{self._tank_config.name} recovered to {level:.1f}%" + ) + self._alarm_since = None + self._alarm_logged = False + + self._set_value(alarm_path, 0) + self._set_value(ack_path, 0) + self._set_value(snooze_path, 0) + + def _handle_changed_value(self, path: str, value: Any) -> bool: + """Accept external writes for alarm acknowledgement and snooze paths.""" + writable = { + '/Alarms/LowLevelAck', '/Alarms/LowLevelSnoozeUntil', + '/Alarms/HighLevelAck', '/Alarms/HighLevelSnoozeUntil', + } + if path in writable: + logger.info(f"External write: {self._tank_config.name} {path} = {value}") + return True + return super()._handle_changed_value(path, value) + + +def create_tank_services( + sensor_data: SensorData, + tank_ids: Optional[List[int]] = None, + start_instance: int = 0, +) -> List[TankService]: + """Create TankService instances for multiple tanks. + + Args: + sensor_data: SensorData instance to read values from + tank_ids: List of tank IDs to create services for. + If None, creates services for all configured tanks. + start_instance: Starting device instance number + + Returns: + List of TankService instances + """ + if tank_ids is None: + tank_ids = list(TANK_CONFIG.keys()) + + services = [] + for i, tank_id in enumerate(tank_ids): + service = TankService( + sensor_data=sensor_data, + tank_id=tank_id, + device_instance=start_instance + i, + ) + services.append(service) + + return services diff --git a/axiom-nmea/requirements.txt b/axiom-nmea/requirements.txt new file mode 100644 index 0000000..39c739c --- /dev/null +++ b/axiom-nmea/requirements.txt @@ -0,0 +1,11 @@ +# Raymarine NMEA Decoder Dependencies + +# No external dependencies required - using only standard library +# The script uses: +# - socket (UDP multicast) +# - struct (binary parsing) +# - argparse (CLI arguments) +# - json (JSON output) +# - datetime (timestamps) +# - threading (multi-group listener) +# - collections (data buffering) diff --git a/axiom-nmea/samples/.gitkeep b/axiom-nmea/samples/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/axiom-nmea/samples/README.md b/axiom-nmea/samples/README.md new file mode 100644 index 0000000..5f4fe35 --- /dev/null +++ b/axiom-nmea/samples/README.md @@ -0,0 +1,40 @@ +# Sample Packet Captures + +This directory contains sample Raymarine network packet captures for testing and development. + +## Expected Files + +Place your `.pcap` capture files here: + +| File | Description | +|------|-------------| +| `raymarine_sample.pcap` | General sample data with mixed sensor readings | +| `raymarine_sample_TWD_62-70_HDG_29-35.pcap` | Capture with known TWD (62-70°) and Heading (29-35°) | +| `raymarine_sample_twd_69-73.pcap` | Additional wind direction samples (TWD 69-73°) | + +## Creating Captures + +To capture Raymarine network traffic: + +```bash +# Using tcpdump (Linux/macOS) +sudo tcpdump -i eth0 -w samples/raymarine_sample.pcap udp port 2565 + +# Capture from specific multicast group +sudo tcpdump -i eth0 -w samples/raymarine_sample.pcap host 226.192.206.102 +``` + +## Using Captures + +Most debug scripts accept a `--pcap` argument: + +```bash +# From project root +python debug/protobuf_decoder.py --pcap samples/raymarine_sample.pcap +python debug/raymarine_decoder.py --pcap samples/raymarine_sample.pcap +python examples/pcap-to-nmea/pcap_to_nmea.py samples/raymarine_sample.pcap +``` + +## Note + +The `.pcap` files themselves are not committed to git (listed in `.gitignore`) to keep the repository size manageable. You'll need to create your own captures or obtain them separately. diff --git a/dbus-generator-ramp/.gitignore b/dbus-generator-ramp/.gitignore new file mode 100644 index 0000000..5923ff5 --- /dev/null +++ b/dbus-generator-ramp/.gitignore @@ -0,0 +1,24 @@ +# Build artifacts +*.tar.gz +*.sha256 + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg +*.egg-info/ +dist/ +build/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Venus OS runtime (created during installation) +ext/ diff --git a/dbus-generator-ramp/Dockerfile b/dbus-generator-ramp/Dockerfile new file mode 100644 index 0000000..dc1d4ea --- /dev/null +++ b/dbus-generator-ramp/Dockerfile @@ -0,0 +1,47 @@ +# Dockerfile for Generator Current Ramp Controller Development +# +# This provides a development environment with D-Bus support for testing. +# Note: Full integration testing requires a real Venus OS device. +# +# Usage: +# docker-compose build +# docker-compose run --rm dev python -m pytest tests/ +# docker-compose run --rm dev python overload_detector.py # Run unit tests +# + +FROM python:3.11-slim + +# Install system dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + dbus \ + libdbus-1-dev \ + libdbus-glib-1-dev \ + libgirepository1.0-dev \ + gcc \ + pkg-config \ + python3-gi \ + python3-gi-cairo \ + gir1.2-glib-2.0 \ + && rm -rf /var/lib/apt/lists/* + +# Install Python dependencies +RUN pip install --no-cache-dir \ + dbus-python \ + PyGObject \ + numpy \ + pytest + +# Create working directory +WORKDIR /app + +# Copy application code +COPY . /app/ + +# Create ext directory for velib_python (will be mounted or mocked) +RUN mkdir -p /app/ext/velib_python + +# Set Python path +ENV PYTHONPATH="/app:/app/ext/velib_python" + +# Default command: run tests +CMD ["python", "-m", "pytest", "-v"] diff --git a/dbus-generator-ramp/README.md b/dbus-generator-ramp/README.md new file mode 100644 index 0000000..4b12b2a --- /dev/null +++ b/dbus-generator-ramp/README.md @@ -0,0 +1,462 @@ +# Generator Current Ramp Controller for Venus OS + +A Venus OS addon that dynamically controls the inverter/charger input current limit when running on generator power, preventing generator overload through intelligent ramping, automatic rollback, and adaptive learning. + +## Features + +- **Preemptive Protection**: Sets 40A limit when generator warm-up is detected +- **Gradual Ramp**: Increases from 40A to 50A over 30 minutes +- **Overload Detection**: Monitors power fluctuations to detect generator stress +- **Fast Recovery**: Quickly ramps back to near-overload point, then slow ramps from there +- **Rapid Overload Protection**: Increases safety margins when overloads occur in quick succession +- **Output Power Correlation**: Learns relationship between output loads and safe input current +- **Persistent Learning**: Model survives reboots, continuously improves over time + +## How It Works + +### Basic Flow + +``` +Generator Starts → Warm-up (40A) → AC Connects → Ramp 40A→50A → Stable + ↓ + Overload detected + ↓ + Rollback to 40A + ↓ + 5 min cooldown + ↓ + FAST ramp to (overload - 4A) + ↓ + SLOW ramp to recovery target +``` + +### Fast Recovery Algorithm + +When overload is detected, instead of slowly ramping from 40A all the way back up: + +1. **Immediate rollback** to 40A (safety) +2. **5-minute cooldown** to let generator stabilize +3. **Fast ramp phase**: Quickly ramp at 5 A/min to `(overload_point - 4A)` +4. **Slow ramp phase**: Then ramp at 0.5 A/min to the recovery target + +**Example**: Overload at 48A + +- Fast ramp: 40A → 44A in ~48 seconds +- Slow ramp: 44A → 46A (recovery target) in ~4 minutes + +### Rapid Overload Protection + +If overload occurs again within 2 minutes, margins increase: + + +| Overload | Fast Ramp Margin | Recovery Margin | +| ----------- | ---------------- | --------------- | +| 1st | 4A | 2A | +| 2nd (rapid) | 6A | 4A | +| 3rd (rapid) | 8A | 6A | + + +This prevents repeated overload cycles by being progressively more conservative. + +### Output Power Correlation Learning + +The system learns that **higher output loads allow higher input current**. + +**Why?** When external loads (AC output) are high, power flows directly through to loads. When loads are low, all input power goes to battery charging, stressing the inverter more. + +**Model**: `max_input_current = base + (slope × output_power) + zone_offset` + + +| Output Power | Zone | Offset | Example Max Input | +| ------------ | ---- | ------ | ----------------- | +| 0-2000W | LOW | -2A | 44 + 1 - 2 = 43A | +| 2000-4000W | MED | 0A | 44 + 3 + 0 = 47A | +| 4000-8000W | HIGH | +4A | 44 + 6 + 4 = 54A | + + +The model learns from: + +- **Overload events**: Strong signal that limit was too high for that output level +- **Stable operation**: Confirms current limit is safe at that output level + +Model parameters persist to `/data/dbus-generator-ramp/learned_model.json`. + +### State Machine + + +| State | Description | +| -------- | -------------------------------------------------------- | +| IDLE | Waiting for generator to start | +| WARMUP | Generator warming up, AC not connected, limit set to 40A | +| RAMPING | Gradually increasing current from 40A to target | +| COOLDOWN | Waiting at 40A after overload (5 minutes) | +| RECOVERY | Fast ramp then slow ramp back up after overload | +| STABLE | At target, monitoring for overload | + + +### Overload Detection + +The detector uses two methods that must both agree: + +1. **Rate-of-Change Reversals**: Counts rapid sign changes in power derivative +2. **Detrended Standard Deviation**: Measures oscillation amplitude after removing smooth trends + +This combination: + +- ✅ Detects: Erratic oscillations (generator overload) +- ❌ Ignores: Smooth load increases/decreases (normal operation) + +## Installation + +### Prerequisites + +- Venus OS v3.10 or later (for warm-up/cool-down support) +- VE.Bus firmware 415 or later +- SSH/root access to the Cerbo GX + +### Steps + +1. **Copy files to Venus OS**: + ```bash + # From your local machine + scp -r dbus-generator-ramp root@:/data/ + ``` +2. **Make scripts executable**: + ```bash + ssh root@ + chmod +x /data/dbus-generator-ramp/service/run + chmod +x /data/dbus-generator-ramp/service/log/run + ``` +3. **Create symlink to velib_python**: + ```bash + ln -s /opt/victronenergy/velib_python /data/dbus-generator-ramp/ext/velib_python + ``` +4. **Create service symlink**: + ```bash + ln -s /data/dbus-generator-ramp/service /opt/victronenergy/service/dbus-generator-ramp + ``` +5. **The service will start automatically** (managed by daemontools) + +### Persistence Across Firmware Updates + +Add to `/data/rc.local`: + +```bash +#!/bin/bash + +# Restore generator ramp controller service link +if [ ! -L /opt/victronenergy/service/dbus-generator-ramp ]; then + ln -s /data/dbus-generator-ramp/service /opt/victronenergy/service/dbus-generator-ramp +fi +``` + +## Configuration + +Edit `/data/dbus-generator-ramp/config.py` to adjust: + +### D-Bus Services + +```python +DBUS_CONFIG = { + 'vebus_service': 'com.victronenergy.vebus.ttyS4', # Your VE.Bus service + 'generator_service': 'com.victronenergy.generator.startstop0', +} +``` + +### Current Limits + +```python +RAMP_CONFIG = { + 'initial_current': 40.0, # Starting current (A) + 'target_current': 50.0, # Target current (A) + 'initial_ramp_rate': 0.333, # A/min (10A over 30 min) + 'recovery_ramp_rate': 0.5, # A/min during recovery + 'cooldown_duration': 300, # Seconds at 40A after overload + + # Fast recovery settings + 'fast_recovery_margin': 4.0, # Amps below overload point + 'fast_ramp_rate': 5.0, # A/min during fast recovery + 'rapid_overload_threshold': 120, # Seconds - rapid if within this + 'rapid_overload_extra_margin': 2.0, # Extra margin per rapid overload +} +``` + +### Output Power Learning + +```python +LEARNING_CONFIG = { + 'enabled': True, + 'power_zones': { + 'LOW': (0, 2000, -2.0), # (min_W, max_W, offset_A) + 'MEDIUM': (2000, 4000, 0.0), + 'HIGH': (4000, 8000, 4.0), + }, + 'initial_base_current': 44.0, # Starting base for model + 'initial_slope': 0.001, # Amps per Watt + 'learning_rate': 0.1, # How fast model adapts + 'min_confidence': 5, # Data points before using model +} +``` + +### Overload Detection + +```python +OVERLOAD_CONFIG = { + 'derivative_threshold': 150, # Min power change to count (W) + 'reversal_threshold': 5, # Reversals to trigger + 'std_dev_threshold': 250, # Std dev threshold (W) + 'confirmation_threshold': 6, # Confirmations needed +} +``` + +## Monitoring + +### D-Bus Paths + +The service publishes status to `com.victronenergy.generatorramp`: + +**Core Status** + + +| Path | Description | +| ----------------- | ----------------------------------------------------------------------------- | +| `/State` | Current state (0=Idle, 1=Warmup, 2=Ramping, 3=Cooldown, 4=Recovery, 5=Stable) | +| `/CurrentLimit` | Current input limit (A) | +| `/TargetLimit` | Target for current ramp (A) | +| `/RecoveryTarget` | Conservative target after overload (A) | +| `/OverloadCount` | Total overloads this session | + + +**Power Monitoring** + + +| Path | Description | +| ---------------------------------- | ------------------------ | +| `/Power/L1`, `/L2`, `/Total` | Input power (W) | +| `/OutputPower/L1`, `/L2`, `/Total` | Output power / loads (W) | + + +**Fast Recovery Status** + + +| Path | Description | +| ------------------------------ | ----------------------------------- | +| `/Recovery/InFastRamp` | 1 if currently in fast ramp phase | +| `/Recovery/FastRampTarget` | Target for fast ramp (A) | +| `/Recovery/RapidOverloadCount` | Count of rapid successive overloads | + + +**Learning Model Status** + + +| Path | Description | +| --------------------------- | ---------------------------------------------- | +| `/Learning/Confidence` | Model confidence (data points) | +| `/Learning/ConfidenceLevel` | LOW, MEDIUM, or HIGH | +| `/Learning/BaseCurrent` | Learned base current (A) | +| `/Learning/SuggestedLimit` | Model's suggested limit for current output (A) | +| `/Learning/DataPoints` | Number of stable operation data points | +| `/Learning/OverloadPoints` | Number of recorded overload events | + + +### View Logs + +```bash +# Live log +tail -F /var/log/dbus-generator-ramp/current | tai64nlocal + +# Recent log +cat /var/log/dbus-generator-ramp/current | tai64nlocal +``` + +### Service Control + +```bash +# Stop service +svc -d /service/dbus-generator-ramp + +# Start service +svc -u /service/dbus-generator-ramp + +# Restart service +svc -t /service/dbus-generator-ramp + +# Check status +svstat /service/dbus-generator-ramp +``` + +### Manual Testing + +```bash +# Check generator state +dbus -y com.victronenergy.generator.startstop0 /State GetValue + +# Check current limit +dbus -y com.victronenergy.vebus.ttyS4 /Ac/In/1/CurrentLimit GetValue + +# Check input power +dbus -y com.victronenergy.vebus.ttyS4 /Ac/ActiveIn/L1/P GetValue +dbus -y com.victronenergy.vebus.ttyS4 /Ac/ActiveIn/L2/P GetValue + +# Check output power (loads) +dbus -y com.victronenergy.vebus.ttyS4 /Ac/Out/L1/P GetValue +dbus -y com.victronenergy.vebus.ttyS4 /Ac/Out/L2/P GetValue + +# Check learning model status +dbus -y com.victronenergy.generatorramp /Learning/SuggestedLimit GetValue +dbus -y com.victronenergy.generatorramp /Learning/Confidence GetValue +``` + +## Native GUI Integration (Optional) + +Add a menu item to the Cerbo's built-in GUI: + +```bash +# Install GUI modifications +./install_gui.sh + +# Remove GUI modifications +./install_gui.sh --remove +``` + +This adds **Settings → Generator start/stop → Dynamic current ramp** with: + +- Enable/disable toggle +- Current status display +- All configurable settings + +**Note**: GUI changes are lost on firmware update. Run `./install_gui.sh` again after updating. + +## Web UI (Optional) + +A browser-based interface at `http://:8088`: + +```bash +# Install with web UI +./install.sh --webui +``` + +## Tuning + +### Overload Detection + +The default thresholds may need adjustment for your specific generator: + +1. **Collect Data**: Log power readings during normal operation and overload + ```bash + # Add to config.py: LOGGING_CONFIG['level'] = 'DEBUG' + ``` +2. **Analyze Patterns**: + - Normal: Low std dev, few reversals + - Overload: High std dev, many reversals +3. **Adjust Thresholds**: + - If false positives: Increase thresholds + - If missed overloads: Decrease thresholds + +### Learning Model + +To tune the power correlation model based on your system: + +1. **Observe stable operation** at different output power levels +2. **Record what input current works** at each level +3. **Adjust `LEARNING_CONFIG`**: + - `initial_base_current`: Max input at 0W output + - `initial_slope`: How much more input per Watt of output + - `power_zones`: Offsets for different load ranges + +**Example tuning**: If you can run 54A stable at 6kW output: + +``` +54 = base + (0.001 × 6000) + 4 (HIGH zone) +54 = base + 6 + 4 +base = 44A +``` + +## Troubleshooting + +### Service Won't Start + +```bash +# Check for Python errors +python3 /data/dbus-generator-ramp/dbus-generator-ramp.py + +# Check D-Bus services exist +dbus -y | grep vebus +dbus -y | grep generator +``` + +### Current Limit Not Changing + +```bash +# Check if adjustable +dbus -y com.victronenergy.vebus.ttyS4 /Ac/In/1/CurrentLimitIsAdjustable GetValue +# Should return 1 +``` + +### Generator State Not Detected + +```bash +# Monitor generator state changes +watch -n 1 'dbus -y com.victronenergy.generator.startstop0 /State GetValue' +``` + +### Reset Learning Model + +```bash +# Delete the learned model to start fresh +rm /data/dbus-generator-ramp/learned_model.json +svc -t /service/dbus-generator-ramp +``` + +## Development + +### Local Testing (without Venus OS) + +```bash +# Build development environment +docker-compose build + +# Run component tests +docker-compose run --rm dev python overload_detector.py +docker-compose run --rm dev python ramp_controller.py + +# Interactive shell +docker-compose run --rm dev bash +``` + +### Running Tests + +```bash +python3 overload_detector.py # Runs built-in tests +python3 ramp_controller.py # Runs built-in tests (includes fast recovery) +``` + +## File Structure + +``` +/data/dbus-generator-ramp/ +├── dbus-generator-ramp.py # Main application +├── config.py # Configuration +├── overload_detector.py # Power fluctuation analysis +├── ramp_controller.py # Current ramping + learning model +├── learned_model.json # Persisted learning data (created at runtime) +├── service/ +│ ├── run # daemontools run script +│ └── log/ +│ └── run # multilog script +├── ext/ +│ └── velib_python/ # Symlink to Venus library +├── Dockerfile # Development environment +├── docker-compose.yml # Development compose +└── README.md # This file +``` + +## License + +MIT License - See LICENSE file + +## Acknowledgments + +- Victron Energy for Venus OS and the excellent D-Bus API +- The Victron Community modifications forum + diff --git a/dbus-generator-ramp/build-package.sh b/dbus-generator-ramp/build-package.sh new file mode 100755 index 0000000..e5cb4e5 --- /dev/null +++ b/dbus-generator-ramp/build-package.sh @@ -0,0 +1,190 @@ +#!/bin/bash +# +# Build script for Generator Current Ramp Controller +# +# Creates a tar.gz package that can be: +# 1. Copied to a CerboGX device +# 2. Untarred to /data/ +# 3. Installed by running install.sh +# +# Usage: +# ./build-package.sh # Creates package with default name +# ./build-package.sh --version 1.0.0 # Creates package with version in name +# ./build-package.sh --output /path/ # Specify output directory +# +# Output: dbus-generator-ramp-.tar.gz +# +# Installation on CerboGX: +# scp dbus-generator-ramp-*.tar.gz root@:/data/ +# ssh root@ +# cd /data +# tar -xzf dbus-generator-ramp-*.tar.gz +# cd dbus-generator-ramp +# ./install.sh [--webui] +# + +set -e + +# Script directory (where the source files are) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Default values +VERSION="1.0.0" +OUTPUT_DIR="$SCRIPT_DIR" +PACKAGE_NAME="dbus-generator-ramp" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --version|-v) + VERSION="$2" + shift 2 + ;; + --output|-o) + OUTPUT_DIR="$2" + shift 2 + ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " -v, --version VERSION Set package version (default: 1.0.0)" + echo " -o, --output PATH Output directory (default: script directory)" + echo " -h, --help Show this help message" + echo "" + echo "Example:" + echo " $0 --version 1.2.0 --output ./dist/" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Build timestamp +BUILD_DATE=$(date -u +"%Y-%m-%d %H:%M:%S UTC") +BUILD_TIMESTAMP=$(date +%Y%m%d%H%M%S) + +# Temporary build directory +BUILD_DIR=$(mktemp -d) +PACKAGE_DIR="$BUILD_DIR/$PACKAGE_NAME" + +echo "==================================================" +echo "Building $PACKAGE_NAME package" +echo "==================================================" +echo "Version: $VERSION" +echo "Build date: $BUILD_DATE" +echo "Source: $SCRIPT_DIR" +echo "Output: $OUTPUT_DIR" +echo "" + +# Create package directory structure +echo "1. Creating package structure..." +mkdir -p "$PACKAGE_DIR" +mkdir -p "$PACKAGE_DIR/service/log" +mkdir -p "$PACKAGE_DIR/service-webui/log" +mkdir -p "$PACKAGE_DIR/qml" + +# Copy main Python files +echo "2. Copying application files..." +cp "$SCRIPT_DIR/dbus-generator-ramp.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/config.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/ramp_controller.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/overload_detector.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/web_ui.py" "$PACKAGE_DIR/" + +# Copy service files +echo "3. Copying service files..." +cp "$SCRIPT_DIR/service/run" "$PACKAGE_DIR/service/" +cp "$SCRIPT_DIR/service/log/run" "$PACKAGE_DIR/service/log/" +cp "$SCRIPT_DIR/service-webui/run" "$PACKAGE_DIR/service-webui/" +cp "$SCRIPT_DIR/service-webui/log/run" "$PACKAGE_DIR/service-webui/log/" + +# Copy QML files +echo "4. Copying GUI files..." +cp "$SCRIPT_DIR/qml/PageSettingsGeneratorRamp.qml" "$PACKAGE_DIR/qml/" + +# Copy installation scripts +echo "5. Copying installation scripts..." +cp "$SCRIPT_DIR/install.sh" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/uninstall.sh" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/install_gui.sh" "$PACKAGE_DIR/" + +# Copy documentation +echo "6. Copying documentation..." +cp "$SCRIPT_DIR/README.md" "$PACKAGE_DIR/" + +# Create version file with build info +echo "7. Creating version info..." +cat > "$PACKAGE_DIR/VERSION" << EOF +Package: $PACKAGE_NAME +Version: $VERSION +Build Date: $BUILD_DATE +Build Timestamp: $BUILD_TIMESTAMP + +Installation: + 1. Copy this package to CerboGX: scp $PACKAGE_NAME-$VERSION.tar.gz root@:/data/ + 2. SSH to CerboGX: ssh root@ + 3. Extract: cd /data && tar -xzf $PACKAGE_NAME-$VERSION.tar.gz + 4. Install: cd $PACKAGE_NAME && ./install.sh [--webui] + +For more information, see README.md +EOF + +# Set executable permissions +echo "8. Setting permissions..." +chmod +x "$PACKAGE_DIR/dbus-generator-ramp.py" +chmod +x "$PACKAGE_DIR/web_ui.py" +chmod +x "$PACKAGE_DIR/install.sh" +chmod +x "$PACKAGE_DIR/uninstall.sh" +chmod +x "$PACKAGE_DIR/install_gui.sh" +chmod +x "$PACKAGE_DIR/service/run" +chmod +x "$PACKAGE_DIR/service/log/run" +chmod +x "$PACKAGE_DIR/service-webui/run" +chmod +x "$PACKAGE_DIR/service-webui/log/run" + +# Create output directory if needed +mkdir -p "$OUTPUT_DIR" + +# Create the tar.gz package +TARBALL_NAME="$PACKAGE_NAME-$VERSION.tar.gz" +TARBALL_PATH="$OUTPUT_DIR/$TARBALL_NAME" + +echo "9. Creating package archive..." +cd "$BUILD_DIR" +tar -czf "$TARBALL_PATH" "$PACKAGE_NAME" + +# Calculate checksum +CHECKSUM=$(sha256sum "$TARBALL_PATH" | cut -d' ' -f1) + +# Create checksum file +echo "$CHECKSUM $TARBALL_NAME" > "$OUTPUT_DIR/$TARBALL_NAME.sha256" + +# Clean up +echo "10. Cleaning up..." +rm -rf "$BUILD_DIR" + +# Get file size +FILE_SIZE=$(du -h "$TARBALL_PATH" | cut -f1) + +echo "" +echo "==================================================" +echo "Build complete!" +echo "==================================================" +echo "" +echo "Package: $TARBALL_PATH" +echo "Size: $FILE_SIZE" +echo "SHA256: $CHECKSUM" +echo "" +echo "Installation on CerboGX:" +echo " scp $TARBALL_PATH root@:/data/" +echo " ssh root@" +echo " cd /data" +echo " tar -xzf $TARBALL_NAME" +echo " cd $PACKAGE_NAME" +echo " ./install.sh # Main service only" +echo " ./install.sh --webui # With web UI" +echo "" diff --git a/dbus-generator-ramp/config.py b/dbus-generator-ramp/config.py new file mode 100644 index 0000000..d36e754 --- /dev/null +++ b/dbus-generator-ramp/config.py @@ -0,0 +1,261 @@ +""" +Configuration for Generator Current Ramp Controller + +All tunable parameters in one place for easy adjustment. +""" + +# ============================================================================= +# D-BUS SERVICE CONFIGURATION +# ============================================================================= + +DBUS_CONFIG = { + # VE.Bus inverter/charger service + 'vebus_service': 'com.victronenergy.vebus.ttyS4', + + # Generator start/stop service + 'generator_service': 'com.victronenergy.generator.startstop0', + + # Which AC input the generator is connected to (1 or 2) + 'generator_ac_input': 1, +} + +# ============================================================================= +# GENERATOR STATE VALUES +# ============================================================================= + +GENERATOR_STATE = { + 'STOPPED': 0, + 'RUNNING': 1, + 'WARMUP': 2, + 'COOLDOWN': 3, + 'ERROR': 10, +} + +# ============================================================================= +# CURRENT LIMIT CONFIGURATION +# ============================================================================= + +RAMP_CONFIG = { + # Starting current limit after generator warm-up (Amps) + 'initial_current': 40.0, + + # Target current limit to ramp up to (Amps) + # This is the max achievable at high output loads (2500-5000W) + # Actual limit is constrained by PowerCorrelationModel based on output power + 'target_current': 54.0, + + # Absolute minimum current limit - safety floor (Amps) + 'minimum_current': 30.0, + + # Maximum current limit - will be read from inverter but this is a sanity check + 'maximum_current': 100.0, + + # Initial ramp rate: Amps per minute + # 10A over 30 minutes = 0.333 A/min + 'initial_ramp_rate': 0.333, + + # Recovery ramp rate after overload: Amps per minute (faster) + 'recovery_ramp_rate': 0.5, + + # How long to wait at initial_current after overload before ramping again (seconds) + 'cooldown_duration': 300, # 5 minutes + + # Safety margin below last stable point for recovery target (Amps) + 'recovery_margin': 2.0, + + # Additional margin per overload event (Amps) + 'margin_per_overload': 2.0, + + # Time stable at target before clearing overload history (seconds) + # Allows system to attempt full power again after extended stable operation + 'history_clear_time': 1800, # 30 minutes + + # How often to update the current limit (seconds) + # More frequent = smoother ramp, but more D-Bus writes + 'ramp_update_interval': 30, + + # --- Fast Recovery Settings --- + # When recovering, fast-ramp to (overload_current - fast_recovery_margin) + # then slow-ramp from there to the target + 'fast_recovery_margin': 4.0, # Amps below overload point + + # If overload occurs again within this time, use larger margin + 'rapid_overload_threshold': 120, # 2 minutes + + # Additional margin when overloads happen in rapid succession + 'rapid_overload_extra_margin': 2.0, # Added to fast_recovery_margin + + # Fast ramp rate (Amps per minute) - much faster than normal + 'fast_ramp_rate': 5.0, # 5A per minute = 12 seconds per amp + + # --- Return to prior stable current after overload --- + # If we were stable at current limit for this long, assume overload was + # e.g. a step load (e.g. heater switched on) and recovery target = that current. + 'return_to_stable_after_overload': True, + 'return_to_stable_min_duration': 1800, # Seconds (30 minutes) stable before returning to that current +} + +# ============================================================================= +# OUTPUT POWER CORRELATION LEARNING +# ============================================================================= + +# Output power determines how much generator power smoothing is applied by the +# inverter/charger. Higher output loads pass through more directly, allowing +# higher input current limits. Lower output loads stress the inverter more +# (all input goes to battery charging), requiring lower input limits. +# +# Key insight: Quick INPUT power fluctuations indicate overload/instability +# regardless of output load level - these should always trigger detection. + +LEARNING_CONFIG = { + # Enable/disable learning system + 'enabled': True, + + # How many data points to keep for correlation analysis + 'max_data_points': 100, + + # Minimum stable time before recording a data point (seconds) + 'min_stable_time': 60, + + # Output power zones for adaptive current limits + # Based on observed relationship between output load and achievable input current: + # - Low output (0-1500W): Inverter does most smoothing, ~45A achievable + # - Medium output (1500-2500W): Transitional zone, ~49A achievable + # - High output (2500-5000W): Direct passthrough, ~54A achievable + 'power_zones': { + # (min_output_watts, max_output_watts): max_input_current_offset + # Offset is added to base learned limit (45A) + 'LOW': (0, 1500, 0.0), # Low loads: 45A limit + 'MEDIUM': (1500, 2500, 4.0), # Transitional: 49A limit + 'HIGH': (2500, 5000, 9.0), # High loads: 54A limit + 'VERY_HIGH': (5000, 10000, 9.0), # Very high: cap at 54A (same as HIGH) + }, + + # Initial model parameters (linear: max_current = base + slope * output_power) + # Base current is achievable at LOW output (0-1500W) + # Slope is 0 because we use discrete zones for step changes + 'initial_base_current': 45.0, + 'initial_slope': 0.0, # Use zones for discrete steps instead of continuous slope + + # Learning rate for updating model (0-1, higher = faster adaptation) + 'learning_rate': 0.1, + + # Minimum confidence (data points) before using learned model + 'min_confidence': 5, + + # --- Output Power Change Detection --- + # If output power increases by this much, re-evaluate recovery target + # Higher output = more power passes through to loads = less inverter stress + 'output_power_increase_threshold': 2000, # Watts + + # Minimum time between target re-evaluations (prevents rapid changes) + 'min_reevaluation_interval': 60, # Seconds +} + +# ============================================================================= +# OVERLOAD DETECTION CONFIGURATION +# ============================================================================= + +# CRITICAL: Overload/instability is detected via INPUT power fluctuations. +# Quick fluctuations on INPUT (not output) indicate generator stress. +# Output power only affects the achievable input current limit, NOT detection. +# Any quick INPUT fluctuations at ANY output load level should trigger detection. + +OVERLOAD_CONFIG = { + # Sampling interval for power readings (milliseconds) + 'sample_interval_ms': 500, + + # --- Method 1: Rate of Change Reversal Detection --- + # Detects rapid oscillations in INPUT power (sign changes in derivative) + # Minimum power change to be considered significant (Watts) + 'derivative_threshold': 150, + + # Number of significant reversals in window to trigger + 'reversal_threshold': 5, + + # Window size for reversal counting (samples) + 'reversal_window_size': 20, # 10 seconds at 500ms + + # --- Method 2: Detrended Standard Deviation --- + # Detects erratic INPUT power fluctuations after removing trend + # Standard deviation threshold after removing linear trend (Watts) + # Note: 400W threshold avoids false positives during normal ramp settling + 'std_dev_threshold': 400, + + # Window size for std dev calculation (samples) + 'std_dev_window_size': 40, # 20 seconds at 500ms + + # --- Confirmation (prevents false positives) --- + # How many samples to consider for confirmation + 'confirmation_window': 10, + + # How many positive detections needed to confirm overload + # Note: 7/10 threshold requires more persistent detection to reduce false positives + 'confirmation_threshold': 7, + + # --- Lockout after detection --- + # Prevent re-detection for this many seconds after triggering + 'lockout_duration': 10, + + # --- Ramp Start Grace Period --- + # After ramp starts, suppress detection for this many seconds + # Allows system to settle without triggering false overload detection + 'ramp_start_grace_period': 30, + + # --- Minimum Output Power Requirement --- + # IMPORTANT: Set to 0 to detect fluctuations at ANY output load level + # Quick INPUT fluctuations indicate overload/instability regardless of output + # Output power only affects achievable limit, not detection sensitivity + 'min_output_power_for_overload': 0, # CRITICAL: Must be 0 - detect at any load + + # --- Trend Direction Check --- + # If power is trending DOWN faster than this rate, ignore as load drop (not overload) + # A true overload has power oscillating at/near ceiling, not dropping + # Watts per second - negative values mean power is dropping + 'trend_drop_threshold': -100, # -100W/s = ignore if dropping faster than this + + # --- Input Power Drop Check --- + # If INPUT power is significantly below recent maximum, it's not an overload + # True overloads oscillate AT/NEAR a ceiling - not dropping away from it + # This catches step-change load drops that trend detection might miss + # Note: We check INPUT power (generator load), NOT output power + # Output power is unreliable because charger can absorb freed capacity + # (e.g., load turns off but charger takes over - input stays high) + 'max_power_drop_for_overload': 1000, # Watts - ignore if INPUT dropped more than this + + # --- Smoothing for trend detection (optional) --- + # If true, apply EMA smoothing before derivative calculation + 'use_smoothing': False, + 'smoothing_alpha': 0.3, +} + +# ============================================================================= +# LOGGING CONFIGURATION +# ============================================================================= + +LOGGING_CONFIG = { + # Log level: DEBUG, INFO, WARNING, ERROR + 'level': 'INFO', + + # Log to console (stdout) + 'console': True, + + # Log file path (set to None to disable file logging) + 'file_path': None, # Venus uses multilog, so we log to stdout + + # Include timestamps in console output (multilog adds its own) + 'include_timestamp': False, +} + +# ============================================================================= +# MAIN LOOP TIMING +# ============================================================================= + +TIMING_CONFIG = { + # Main loop interval (milliseconds) + # This controls how often we check state and update + 'main_loop_interval_ms': 500, + + # Timeout for D-Bus operations (seconds) + 'dbus_timeout': 5, +} diff --git a/dbus-generator-ramp/dbus-generator-ramp.py b/dbus-generator-ramp/dbus-generator-ramp.py new file mode 100755 index 0000000..9eace68 --- /dev/null +++ b/dbus-generator-ramp/dbus-generator-ramp.py @@ -0,0 +1,967 @@ +#!/usr/bin/env python3 +""" +Venus OS Generator Current Ramp Controller + +Monitors generator operation and dynamically adjusts inverter/charger +input current limit to prevent generator overload. + +This version publishes its own D-Bus service so status is visible via MQTT: + N//generatorramp/0/... + +Features: +- Preemptively sets 40A limit when generator enters warm-up +- Ramps from 40A to 50A over 30 minutes after AC connects +- Detects generator overload via power fluctuation analysis +- Rolls back to 40A on overload, then conservatively ramps back up +- Publishes status to D-Bus/MQTT for monitoring +- Settings adjustable via D-Bus/MQTT + +Author: Claude (Anthropic) +License: MIT +""" + +import sys +import os +import logging +import signal +from time import time, sleep + +# Add velib_python to path (Venus OS standard location) +sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'ext', 'velib_python')) +sys.path.insert(1, '/opt/victronenergy/velib_python') + +try: + from gi.repository import GLib +except ImportError: + print("ERROR: GLib not available. This script must run on Venus OS.") + sys.exit(1) + +try: + import dbus + from dbus.mainloop.glib import DBusGMainLoop + from vedbus import VeDbusService + from settingsdevice import SettingsDevice +except ImportError as e: + print(f"ERROR: Required module not available: {e}") + print("This script must run on Venus OS.") + sys.exit(1) + +from config import ( + DBUS_CONFIG, GENERATOR_STATE, RAMP_CONFIG, + OVERLOAD_CONFIG, LOGGING_CONFIG, TIMING_CONFIG, LEARNING_CONFIG +) +from overload_detector import OverloadDetector +from ramp_controller import RampController + + +# Version +VERSION = '1.1.1' + +# D-Bus service name for our addon +SERVICE_NAME = 'com.victronenergy.generatorramp' + + +class GeneratorRampController: + """ + Main controller that coordinates: + - D-Bus monitoring of generator and inverter state + - Overload detection from power readings + - Current limit ramping + - Publishing status to D-Bus (visible via MQTT) + """ + + # Controller states + STATE_IDLE = 0 + STATE_WARMUP = 1 + STATE_RAMPING = 2 + STATE_COOLDOWN = 3 + STATE_RECOVERY = 4 + STATE_STABLE = 5 + + STATE_NAMES = { + 0: 'Idle', + 1: 'Warm-up', + 2: 'Ramping', + 3: 'Cooldown', + 4: 'Recovery', + 5: 'Stable', + } + + def __init__(self): + self._setup_logging() + self.logger = logging.getLogger('GenRampCtrl') + self.logger.info(f"Initializing Generator Ramp Controller v{VERSION}") + + # Components + self.overload_detector = OverloadDetector(OVERLOAD_CONFIG) + self.ramp_controller = RampController(RAMP_CONFIG) + + # State + self.state = self.STATE_IDLE + self.state_enter_time = time() + + # Cached values from D-Bus + self.generator_state = GENERATOR_STATE['STOPPED'] + self.ac_connected = False + self.current_l1_power = 0 + self.current_l2_power = 0 + self.current_limit_setting = 0 + + # Output power tracking (loads on inverter output) + self.output_l1_power = 0 + self.output_l2_power = 0 + + # Enabled flag + self.enabled = True + + # D-Bus connection + self.bus = dbus.SystemBus() + + # Create our D-Bus service for publishing status + self._create_dbus_service() + + # Set up settings (stored in Venus localsettings) + self._setup_settings() + + # Connect to VE.Bus and Generator services + self._init_dbus_monitors() + + # Start main loop timer + interval_ms = TIMING_CONFIG['main_loop_interval_ms'] + GLib.timeout_add(interval_ms, self._main_loop) + + self.logger.info(f"Initialized. Main loop interval: {interval_ms}ms") + + def _setup_logging(self): + """Configure logging based on config""" + level = getattr(logging, LOGGING_CONFIG['level'], logging.INFO) + + if LOGGING_CONFIG['include_timestamp']: + fmt = '%(asctime)s %(levelname)s %(name)s: %(message)s' + else: + fmt = '%(levelname)s %(name)s: %(message)s' + + logging.basicConfig( + level=level, + format=fmt, + stream=sys.stdout + ) + + def _create_dbus_service(self): + """Create our own D-Bus service for publishing status""" + self.logger.info(f"Creating D-Bus service: {SERVICE_NAME}") + + # Retry logic in case previous instance hasn't released the bus name yet + max_retries = 5 + retry_delay = 1.0 # seconds + + for attempt in range(max_retries): + try: + self.dbus_service = VeDbusService(SERVICE_NAME, self.bus) + break # Success + except dbus.exceptions.NameExistsException: + if attempt < max_retries - 1: + self.logger.warning( + f"D-Bus name exists, retrying in {retry_delay}s " + f"(attempt {attempt + 1}/{max_retries})" + ) + sleep(retry_delay) + retry_delay *= 2 # Exponential backoff + else: + self.logger.error("Failed to acquire D-Bus name after retries") + raise + + # Add management paths (required for Venus) + self.dbus_service.add_path('/Mgmt/ProcessName', 'dbus-generator-ramp') + self.dbus_service.add_path('/Mgmt/ProcessVersion', VERSION) + self.dbus_service.add_path('/Mgmt/Connection', 'local') + + # Add device info + self.dbus_service.add_path('/DeviceInstance', 0) + self.dbus_service.add_path('/ProductId', 0xFFFF) + self.dbus_service.add_path('/ProductName', 'Generator Ramp Controller') + self.dbus_service.add_path('/FirmwareVersion', VERSION) + self.dbus_service.add_path('/Connected', 1) + + # Status paths (read-only) - these will be visible in MQTT + self.dbus_service.add_path('/State', self.STATE_IDLE, + gettextcallback=lambda p, v: self.STATE_NAMES.get(v, 'Unknown')) + self.dbus_service.add_path('/CurrentLimit', 0.0, + gettextcallback=lambda p, v: f"{v:.1f}A") + self.dbus_service.add_path('/TargetLimit', RAMP_CONFIG['target_current'], + gettextcallback=lambda p, v: f"{v:.1f}A") + self.dbus_service.add_path('/RecoveryTarget', RAMP_CONFIG['target_current'], + gettextcallback=lambda p, v: f"{v:.1f}A") + self.dbus_service.add_path('/OverloadCount', 0) + self.dbus_service.add_path('/LastStableCurrent', RAMP_CONFIG['initial_current'], + gettextcallback=lambda p, v: f"{v:.1f}A") + + # Input power monitoring + self.dbus_service.add_path('/Power/L1', 0.0, + gettextcallback=lambda p, v: f"{v:.0f}W") + self.dbus_service.add_path('/Power/L2', 0.0, + gettextcallback=lambda p, v: f"{v:.0f}W") + self.dbus_service.add_path('/Power/Total', 0.0, + gettextcallback=lambda p, v: f"{v:.0f}W") + + # Output power monitoring (loads on inverter output) + self.dbus_service.add_path('/OutputPower/L1', 0.0, + gettextcallback=lambda p, v: f"{v:.0f}W") + self.dbus_service.add_path('/OutputPower/L2', 0.0, + gettextcallback=lambda p, v: f"{v:.0f}W") + self.dbus_service.add_path('/OutputPower/Total', 0.0, + gettextcallback=lambda p, v: f"{v:.0f}W") + + # Overload detection diagnostics + self.dbus_service.add_path('/Detection/Reversals', 0) + self.dbus_service.add_path('/Detection/StdDev', 0.0, + gettextcallback=lambda p, v: f"{v:.1f}W") + self.dbus_service.add_path('/Detection/IsOverload', 0) + self.dbus_service.add_path('/Detection/OutputPowerOk', 1) # 1=sufficient load for detection + self.dbus_service.add_path('/Detection/Trend', 0.0, + gettextcallback=lambda p, v: f"{v:.1f}W/s") + self.dbus_service.add_path('/Detection/TrendOk', 1) # 1=not dropping fast + self.dbus_service.add_path('/Detection/PowerDrop', 0.0, + gettextcallback=lambda p, v: f"{v:.0f}W") + self.dbus_service.add_path('/Detection/PowerDropOk', 1) # 1=input not dropped from ceiling + + # Generator and AC status (mirrors) + self.dbus_service.add_path('/Generator/State', 0, + gettextcallback=self._generator_state_text) + self.dbus_service.add_path('/AcInput/Connected', 0) + + # Ramp progress + self.dbus_service.add_path('/Ramp/Progress', 0, + gettextcallback=lambda p, v: f"{v}%") + self.dbus_service.add_path('/Ramp/TimeRemaining', 0, + gettextcallback=lambda p, v: f"{v//60}m {v%60}s" if v > 0 else "0s") + + # Fast recovery status + self.dbus_service.add_path('/Recovery/InFastRamp', 0) + self.dbus_service.add_path('/Recovery/FastRampTarget', 0.0, + gettextcallback=lambda p, v: f"{v:.1f}A") + self.dbus_service.add_path('/Recovery/RapidOverloadCount', 0) + + # Learning model status + self.dbus_service.add_path('/Learning/Confidence', 0) + self.dbus_service.add_path('/Learning/ConfidenceLevel', 'LOW') + self.dbus_service.add_path('/Learning/BaseCurrent', LEARNING_CONFIG['initial_base_current'], + gettextcallback=lambda p, v: f"{v:.1f}A") + self.dbus_service.add_path('/Learning/SuggestedLimit', RAMP_CONFIG['target_current'], + gettextcallback=lambda p, v: f"{v:.1f}A") + self.dbus_service.add_path('/Learning/DataPoints', 0) + self.dbus_service.add_path('/Learning/OverloadPoints', 0) + + # Writable settings (can be changed via D-Bus/MQTT) + self.dbus_service.add_path('/Settings/InitialCurrent', + RAMP_CONFIG['initial_current'], + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: f"{v:.1f}A") + self.dbus_service.add_path('/Settings/TargetCurrent', + RAMP_CONFIG['target_current'], + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: f"{v:.1f}A") + self.dbus_service.add_path('/Settings/RampDuration', + 30, # minutes + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: f"{v} min") + self.dbus_service.add_path('/Settings/CooldownDuration', + int(RAMP_CONFIG['cooldown_duration'] // 60), + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: f"{v} min") + self.dbus_service.add_path('/Settings/Enabled', + 1, + writeable=True, + onchangecallback=self._on_setting_changed) + + # Power zone settings (output-based input current limits) + # These control how input current limit varies with output load + self.dbus_service.add_path('/Settings/LowOutputLimit', + LEARNING_CONFIG['initial_base_current'], + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: f"{v:.0f}A") + self.dbus_service.add_path('/Settings/MediumOutputLimit', + LEARNING_CONFIG['initial_base_current'] + LEARNING_CONFIG['power_zones']['MEDIUM'][2], + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: f"{v:.0f}A") + self.dbus_service.add_path('/Settings/HighOutputLimit', + LEARNING_CONFIG['initial_base_current'] + LEARNING_CONFIG['power_zones']['HIGH'][2], + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: f"{v:.0f}A") + self.dbus_service.add_path('/Settings/LowOutputThreshold', + LEARNING_CONFIG['power_zones']['LOW'][1], + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: f"{v:.0f}W") + self.dbus_service.add_path('/Settings/HighOutputThreshold', + LEARNING_CONFIG['power_zones']['MEDIUM'][1], + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: f"{v:.0f}W") + + # Return to prior stable current after overload (if stable > N minutes) + self.dbus_service.add_path('/Settings/ReturnToStableAfterOverload', + 1 if RAMP_CONFIG.get('return_to_stable_after_overload', True) else 0, + writeable=True, + onchangecallback=self._on_setting_changed) + self.dbus_service.add_path('/Settings/ReturnToStableMinMinutes', + int(RAMP_CONFIG.get('return_to_stable_min_duration', 1800) // 60), + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: f"{v} min") + + self.logger.info("D-Bus service created") + + def _generator_state_text(self, path, value): + """Get text for generator state""" + states = {0: 'Stopped', 1: 'Running', 2: 'Warm-up', 3: 'Cool-down', 10: 'Error'} + return states.get(value, f'Unknown ({value})') + + def _on_setting_changed(self, path, value): + """Handle setting changes from D-Bus/MQTT""" + self.logger.info(f"Setting changed: {path} = {value}") + + if path == '/Settings/InitialCurrent': + RAMP_CONFIG['initial_current'] = float(value) + self._save_setting('InitialCurrent', float(value)) + + elif path == '/Settings/TargetCurrent': + RAMP_CONFIG['target_current'] = float(value) + self.dbus_service['/TargetLimit'] = float(value) + self._save_setting('TargetCurrent', float(value)) + + elif path == '/Settings/RampDuration': + duration_min = int(value) + delta = RAMP_CONFIG['target_current'] - RAMP_CONFIG['initial_current'] + RAMP_CONFIG['initial_ramp_rate'] = delta / duration_min if duration_min > 0 else 0.333 + self._save_setting('RampDuration', duration_min) + + elif path == '/Settings/CooldownDuration': + RAMP_CONFIG['cooldown_duration'] = int(value) * 60 + self._save_setting('CooldownDuration', int(value)) + + elif path == '/Settings/Enabled': + self.enabled = bool(value) + self._save_setting('Enabled', int(value)) + if not self.enabled: + self.logger.info("Controller disabled") + self._transition_to(self.STATE_IDLE) + + # Power zone settings + elif path == '/Settings/LowOutputLimit': + self._update_power_zones(low_limit=float(value)) + self._save_setting('LowOutputLimit', float(value)) + + elif path == '/Settings/MediumOutputLimit': + self._update_power_zones(medium_limit=float(value)) + self._save_setting('MediumOutputLimit', float(value)) + + elif path == '/Settings/HighOutputLimit': + self._update_power_zones(high_limit=float(value)) + self._save_setting('HighOutputLimit', float(value)) + + elif path == '/Settings/LowOutputThreshold': + self._update_power_zones(low_threshold=int(value)) + self._save_setting('LowOutputThreshold', int(value)) + + elif path == '/Settings/HighOutputThreshold': + self._update_power_zones(high_threshold=int(value)) + self._save_setting('HighOutputThreshold', int(value)) + + elif path == '/Settings/ReturnToStableAfterOverload': + RAMP_CONFIG['return_to_stable_after_overload'] = bool(value) + self._save_setting('ReturnToStableAfterOverload', int(value)) + + elif path == '/Settings/ReturnToStableMinMinutes': + RAMP_CONFIG['return_to_stable_min_duration'] = int(value) * 60 + self._save_setting('ReturnToStableMinMinutes', int(value)) + + return True + + def _save_setting(self, name, value): + """Save a setting to localsettings""" + if self.settings: + try: + self.settings[name] = value + except Exception as e: + self.logger.warning(f"Failed to save setting {name}: {e}") + + def _update_power_zones(self, low_limit=None, medium_limit=None, high_limit=None, + low_threshold=None, high_threshold=None): + """ + Update power zone configuration based on UI settings. + + The UI exposes absolute current limits for each zone, but internally + we store base_current and zone offsets. This method converts between them. + """ + # Get current values + base = LEARNING_CONFIG['initial_base_current'] + zones = LEARNING_CONFIG['power_zones'] + + # Current absolute limits + current_low = base + zones['LOW'][2] + current_medium = base + zones['MEDIUM'][2] + current_high = base + zones['HIGH'][2] + + # Current thresholds + current_low_thresh = zones['LOW'][1] + current_high_thresh = zones['MEDIUM'][1] + + # Apply changes + if low_limit is not None: + # Low limit becomes the new base + new_base = float(low_limit) + # Adjust other offsets to maintain their absolute values + medium_offset = (medium_limit if medium_limit else current_medium) - new_base + high_offset = (high_limit if high_limit else current_high) - new_base + LEARNING_CONFIG['initial_base_current'] = new_base + zones['LOW'] = (zones['LOW'][0], zones['LOW'][1], 0.0) + zones['MEDIUM'] = (zones['MEDIUM'][0], zones['MEDIUM'][1], medium_offset) + zones['HIGH'] = (zones['HIGH'][0], zones['HIGH'][1], high_offset) + zones['VERY_HIGH'] = (zones['VERY_HIGH'][0], zones['VERY_HIGH'][1], high_offset) + # Update the ramp controller's model + self.ramp_controller.power_model.base_current = new_base + self.logger.info(f"Power zones updated: base={new_base}A") + + if medium_limit is not None and low_limit is None: + medium_offset = float(medium_limit) - base + zones['MEDIUM'] = (zones['MEDIUM'][0], zones['MEDIUM'][1], medium_offset) + self.logger.info(f"Medium zone limit: {medium_limit}A (offset={medium_offset}A)") + + if high_limit is not None and low_limit is None: + high_offset = float(high_limit) - base + zones['HIGH'] = (zones['HIGH'][0], zones['HIGH'][1], high_offset) + zones['VERY_HIGH'] = (zones['VERY_HIGH'][0], zones['VERY_HIGH'][1], high_offset) + self.logger.info(f"High zone limit: {high_limit}A (offset={high_offset}A)") + + if low_threshold is not None: + # Update LOW zone upper bound and MEDIUM zone lower bound + zones['LOW'] = (0, int(low_threshold), zones['LOW'][2]) + zones['MEDIUM'] = (int(low_threshold), zones['MEDIUM'][1], zones['MEDIUM'][2]) + self.logger.info(f"Low/Medium threshold: {low_threshold}W") + + if high_threshold is not None: + # Update MEDIUM zone upper bound and HIGH zone lower bound + zones['MEDIUM'] = (zones['MEDIUM'][0], int(high_threshold), zones['MEDIUM'][2]) + zones['HIGH'] = (int(high_threshold), zones['HIGH'][1], zones['HIGH'][2]) + self.logger.info(f"Medium/High threshold: {high_threshold}W") + + def _setup_settings(self): + """Set up persistent settings via Venus localsettings""" + self.settings = None + try: + settings_path = '/Settings/GeneratorRamp' + + # Define settings with defaults [path, default, min, max] + settings_def = { + 'InitialCurrent': [settings_path + '/InitialCurrent', 40.0, 10.0, 100.0], + 'TargetCurrent': [settings_path + '/TargetCurrent', 54.0, 10.0, 100.0], + 'RampDuration': [settings_path + '/RampDuration', 30, 1, 120], + 'CooldownDuration': [settings_path + '/CooldownDuration', 5, 1, 30], + 'Enabled': [settings_path + '/Enabled', 1, 0, 1], + # Return to prior stable current after overload if stable > N min + 'ReturnToStableAfterOverload': [settings_path + '/ReturnToStableAfterOverload', 1, 0, 1], + 'ReturnToStableMinMinutes': [settings_path + '/ReturnToStableMinMinutes', 30, 5, 120], + # Power zone settings (output-based input current limits) + 'LowOutputLimit': [settings_path + '/LowOutputLimit', 45.0, 30.0, 100.0], + 'MediumOutputLimit': [settings_path + '/MediumOutputLimit', 49.0, 30.0, 100.0], + 'HighOutputLimit': [settings_path + '/HighOutputLimit', 54.0, 30.0, 100.0], + 'LowOutputThreshold': [settings_path + '/LowOutputThreshold', 1500, 0, 10000], + 'HighOutputThreshold': [settings_path + '/HighOutputThreshold', 2500, 0, 10000], + } + + self.settings = SettingsDevice( + self.bus, + settings_def, + self._on_persistent_setting_changed + ) + + # Load saved settings + if self.settings: + self._load_settings() + + self.logger.info("Persistent settings initialized") + + except Exception as e: + self.logger.warning(f"Could not initialize persistent settings: {e}") + self.logger.warning("Settings will not persist across restarts") + + def _load_settings(self): + """Load settings from Venus localsettings""" + if not self.settings: + return + + try: + RAMP_CONFIG['initial_current'] = float(self.settings['InitialCurrent']) + RAMP_CONFIG['target_current'] = float(self.settings['TargetCurrent']) + + duration_min = int(self.settings['RampDuration']) + delta = RAMP_CONFIG['target_current'] - RAMP_CONFIG['initial_current'] + RAMP_CONFIG['initial_ramp_rate'] = delta / duration_min if duration_min > 0 else 0.333 + + RAMP_CONFIG['cooldown_duration'] = int(self.settings['CooldownDuration']) * 60 + + self.enabled = bool(self.settings['Enabled']) + + # Update D-Bus paths + self.dbus_service['/Settings/InitialCurrent'] = RAMP_CONFIG['initial_current'] + self.dbus_service['/Settings/TargetCurrent'] = RAMP_CONFIG['target_current'] + self.dbus_service['/Settings/RampDuration'] = duration_min + self.dbus_service['/Settings/CooldownDuration'] = int(self.settings['CooldownDuration']) + self.dbus_service['/Settings/Enabled'] = 1 if self.enabled else 0 + self.dbus_service['/TargetLimit'] = RAMP_CONFIG['target_current'] + + # Return to stable after overload (optional keys for older installs) + RAMP_CONFIG['return_to_stable_after_overload'] = bool( + self.settings.get('ReturnToStableAfterOverload', 1)) + RAMP_CONFIG['return_to_stable_min_duration'] = int( + self.settings.get('ReturnToStableMinMinutes', 30)) * 60 + self.dbus_service['/Settings/ReturnToStableAfterOverload'] = 1 if RAMP_CONFIG['return_to_stable_after_overload'] else 0 + self.dbus_service['/Settings/ReturnToStableMinMinutes'] = RAMP_CONFIG['return_to_stable_min_duration'] // 60 + + # Load power zone settings + low_limit = float(self.settings['LowOutputLimit']) + medium_limit = float(self.settings['MediumOutputLimit']) + high_limit = float(self.settings['HighOutputLimit']) + low_threshold = int(self.settings['LowOutputThreshold']) + high_threshold = int(self.settings['HighOutputThreshold']) + + # Update power zones configuration + self._update_power_zones( + low_limit=low_limit, + medium_limit=medium_limit, + high_limit=high_limit, + low_threshold=low_threshold, + high_threshold=high_threshold + ) + + # Update D-Bus paths for power zones + self.dbus_service['/Settings/LowOutputLimit'] = low_limit + self.dbus_service['/Settings/MediumOutputLimit'] = medium_limit + self.dbus_service['/Settings/HighOutputLimit'] = high_limit + self.dbus_service['/Settings/LowOutputThreshold'] = low_threshold + self.dbus_service['/Settings/HighOutputThreshold'] = high_threshold + + self.logger.info( + f"Loaded settings: {RAMP_CONFIG['initial_current']}A -> " + f"{RAMP_CONFIG['target_current']}A over {duration_min}min" + ) + self.logger.info( + f"Power zones: {low_limit}A (0-{low_threshold}W), " + f"{medium_limit}A ({low_threshold}-{high_threshold}W), " + f"{high_limit}A ({high_threshold}W+)" + ) + + except Exception as e: + self.logger.warning(f"Error loading settings: {e}") + + def _on_persistent_setting_changed(self, setting, old_value, new_value): + """Called when a persistent setting changes externally""" + self.logger.info(f"Persistent setting changed: {setting} = {new_value}") + self._load_settings() + + def _init_dbus_monitors(self): + """Initialize D-Bus service connections""" + try: + self.vebus_service = DBUS_CONFIG['vebus_service'] + self.generator_service = DBUS_CONFIG['generator_service'] + + self.logger.info(f"Monitoring VE.Bus: {self.vebus_service}") + self.logger.info(f"Monitoring Generator: {self.generator_service}") + + # Read initial values + self._read_generator_state() + self._read_ac_state() + self._read_current_limit() + + except dbus.exceptions.DBusException as e: + self.logger.error(f"D-Bus initialization failed: {e}") + raise + + def _get_dbus_value(self, service, path): + """Get a value from D-Bus service""" + try: + obj = self.bus.get_object(service, path, introspect=False) + return obj.GetValue(dbus_interface='com.victronenergy.BusItem') + except dbus.exceptions.DBusException as e: + self.logger.debug(f"Failed to get {service}{path}: {e}") + return None + + def _set_dbus_value(self, service, path, value): + """Set a value on D-Bus service""" + try: + obj = self.bus.get_object(service, path, introspect=False) + # Wrap value in appropriate D-Bus type (variant) + if isinstance(value, float): + dbus_value = dbus.Double(value, variant_level=1) + elif isinstance(value, int): + dbus_value = dbus.Int32(value, variant_level=1) + else: + dbus_value = value + obj.SetValue(dbus_value, dbus_interface='com.victronenergy.BusItem') + self.logger.debug(f"Set {path} = {value}") + return True + except dbus.exceptions.DBusException as e: + self.logger.error(f"Failed to set {path}: {e}") + return False + + def _read_generator_state(self): + """Read generator state from D-Bus""" + value = self._get_dbus_value(self.generator_service, '/State') + self.generator_state = int(value) if value is not None else 0 + self.dbus_service['/Generator/State'] = self.generator_state + + def _read_ac_state(self): + """Read AC input connection state from D-Bus""" + value = self._get_dbus_value(self.vebus_service, '/Ac/ActiveIn/Connected') + self.ac_connected = bool(value) if value is not None else False + self.dbus_service['/AcInput/Connected'] = 1 if self.ac_connected else 0 + + def _read_power(self): + """Read AC input power from D-Bus""" + value = self._get_dbus_value(self.vebus_service, '/Ac/ActiveIn/L1/P') + self.current_l1_power = float(value) if value is not None else 0 + + value = self._get_dbus_value(self.vebus_service, '/Ac/ActiveIn/L2/P') + self.current_l2_power = float(value) if value is not None else 0 + + self.dbus_service['/Power/L1'] = self.current_l1_power + self.dbus_service['/Power/L2'] = self.current_l2_power + self.dbus_service['/Power/Total'] = self.current_l1_power + self.current_l2_power + + def _read_current_limit(self): + """Read current input limit setting from D-Bus""" + value = self._get_dbus_value(self.vebus_service, '/Ac/In/1/CurrentLimit') + self.current_limit_setting = float(value) if value is not None else 0 + self.dbus_service['/CurrentLimit'] = self.current_limit_setting + + def _read_output_power(self): + """Read AC output power (loads) from D-Bus""" + value = self._get_dbus_value(self.vebus_service, '/Ac/Out/L1/P') + self.output_l1_power = float(value) if value is not None else 0 + + value = self._get_dbus_value(self.vebus_service, '/Ac/Out/L2/P') + self.output_l2_power = float(value) if value is not None else 0 + + total_output = self.output_l1_power + self.output_l2_power + + self.dbus_service['/OutputPower/L1'] = self.output_l1_power + self.dbus_service['/OutputPower/L2'] = self.output_l2_power + self.dbus_service['/OutputPower/Total'] = total_output + + # Update ramp controller with current output power + self.ramp_controller.set_output_power(total_output) + + def _set_current_limit(self, limit: float) -> bool: + """Set the input current limit""" + limit = round(limit, 1) + self.logger.info(f"Setting current limit to {limit}A") + success = self._set_dbus_value(self.vebus_service, '/Ac/In/1/CurrentLimit', limit) + if success: + self.dbus_service['/CurrentLimit'] = limit + return success + + def _transition_to(self, new_state: int): + """Transition to a new controller state""" + old_name = self.STATE_NAMES.get(self.state, 'Unknown') + new_name = self.STATE_NAMES.get(new_state, 'Unknown') + self.logger.info(f"State: {old_name} -> {new_name}") + + self.state = new_state + self.state_enter_time = time() + self.dbus_service['/State'] = new_state + + if new_state == self.STATE_IDLE: + self.overload_detector.reset() + self.ramp_controller.reset() + self.dbus_service['/Ramp/Progress'] = 0 + self.dbus_service['/Ramp/TimeRemaining'] = 0 + + def _update_ramp_progress(self): + """Update ramp progress indicators""" + if not self.ramp_controller.is_ramping: + # When stable at target, show 100% + if self.state == self.STATE_STABLE: + self.dbus_service['/Ramp/Progress'] = 100 + self.dbus_service['/Ramp/TimeRemaining'] = 0 + return + + current = self.ramp_controller.current_limit + initial = RAMP_CONFIG['initial_current'] + target = self.ramp_controller.state.target_limit + + if target > initial: + progress = int(100 * (current - initial) / (target - initial)) + progress = max(0, min(100, progress)) + else: + progress = 100 + + self.dbus_service['/Ramp/Progress'] = progress + + if self.ramp_controller.state.is_recovery: + rate = RAMP_CONFIG['recovery_ramp_rate'] + else: + rate = RAMP_CONFIG['initial_ramp_rate'] + + remaining_amps = target - current + if rate > 0: + remaining_seconds = int(remaining_amps / rate * 60) + else: + remaining_seconds = 0 + + self.dbus_service['/Ramp/TimeRemaining'] = max(0, remaining_seconds) + + def _main_loop(self) -> bool: + """Main control loop - called periodically by GLib.""" + try: + now = time() + + # Check if enabled + if not self.enabled: + return True + + # Read current states from D-Bus + self._read_generator_state() + self._read_ac_state() + self._read_power() + self._read_output_power() + self._read_current_limit() + + # Check for generator stop + if self.generator_state == GENERATOR_STATE['STOPPED']: + if self.state != self.STATE_IDLE: + self.logger.info("Generator stopped") + self._transition_to(self.STATE_IDLE) + return True + + # Run overload detection + is_overload = False + if self.ac_connected and self.state in [self.STATE_RAMPING, self.STATE_RECOVERY, self.STATE_STABLE]: + total_output_power = self.output_l1_power + self.output_l2_power + is_overload, diag = self.overload_detector.update( + self.current_l1_power, + self.current_l2_power, + now, + output_power=total_output_power + ) + if 'reversals' in diag: + self.dbus_service['/Detection/Reversals'] = diag.get('reversals', 0) + self.dbus_service['/Detection/StdDev'] = diag.get('std_dev', 0) + self.dbus_service['/Detection/IsOverload'] = 1 if is_overload else 0 + self.dbus_service['/Detection/OutputPowerOk'] = 1 if diag.get('output_power_ok', True) else 0 + self.dbus_service['/Detection/Trend'] = diag.get('trend', 0.0) + self.dbus_service['/Detection/TrendOk'] = 1 if diag.get('trend_ok', True) else 0 + self.dbus_service['/Detection/PowerDrop'] = diag.get('power_drop', 0.0) + self.dbus_service['/Detection/PowerDropOk'] = 1 if diag.get('power_drop_ok', True) else 0 + + # State machine + if self.state == self.STATE_IDLE: + self._handle_idle(now) + elif self.state == self.STATE_WARMUP: + self._handle_warmup(now) + elif self.state == self.STATE_RAMPING: + self._handle_ramping(now, is_overload) + elif self.state == self.STATE_COOLDOWN: + self._handle_cooldown(now) + elif self.state == self.STATE_RECOVERY: + self._handle_recovery(now, is_overload) + elif self.state == self.STATE_STABLE: + self._handle_stable(now, is_overload) + + # Update progress + self._update_ramp_progress() + + # Update status + status = self.ramp_controller.get_status() + self.dbus_service['/RecoveryTarget'] = status['recovery_target'] + self.dbus_service['/OverloadCount'] = status['overload_count'] + self.dbus_service['/LastStableCurrent'] = status['last_stable'] + + # Update fast recovery status + self.dbus_service['/Recovery/InFastRamp'] = 1 if status.get('in_fast_ramp') else 0 + self.dbus_service['/Recovery/FastRampTarget'] = status.get('fast_ramp_target') or 0.0 + self.dbus_service['/Recovery/RapidOverloadCount'] = status.get('rapid_overload_count', 0) + + # Update learning model status + power_model = status.get('power_model', {}) + self.dbus_service['/Learning/Confidence'] = power_model.get('confidence', 0) + self.dbus_service['/Learning/ConfidenceLevel'] = power_model.get('confidence_level', 'LOW') + self.dbus_service['/Learning/BaseCurrent'] = power_model.get('base_current', 42.0) + self.dbus_service['/Learning/SuggestedLimit'] = status.get('suggested_limit', 50.0) + self.dbus_service['/Learning/DataPoints'] = power_model.get('data_points', 0) + self.dbus_service['/Learning/OverloadPoints'] = power_model.get('overload_points', 0) + + except Exception as e: + self.logger.error(f"Main loop error: {e}", exc_info=True) + + return True + + def _handle_idle(self, now: float): + """IDLE: Wait for generator to start warm-up""" + if self.generator_state == GENERATOR_STATE['WARMUP']: + self.logger.info("Generator warm-up detected") + new_limit = self.ramp_controller.set_initial_limit(self.current_limit_setting) + self._set_current_limit(new_limit) + self._transition_to(self.STATE_WARMUP) + + elif self.generator_state == GENERATOR_STATE['RUNNING'] and self.ac_connected: + self.logger.info("Generator already running with AC connected") + new_limit = self.ramp_controller.set_initial_limit(self.current_limit_setting) + self._set_current_limit(new_limit) + self.ramp_controller.start_ramp(now) + self.overload_detector.set_ramp_start(now) + self._transition_to(self.STATE_RAMPING) + + def _handle_warmup(self, now: float): + """WARMUP: Generator warming up, AC not yet connected""" + if self.generator_state == GENERATOR_STATE['RUNNING'] and self.ac_connected: + self.logger.info("Warm-up complete, AC connected - starting ramp") + self.ramp_controller.start_ramp(now) + self.overload_detector.set_ramp_start(now) + self._transition_to(self.STATE_RAMPING) + elif self.generator_state not in [GENERATOR_STATE['WARMUP'], GENERATOR_STATE['RUNNING']]: + self._transition_to(self.STATE_IDLE) + + def _handle_ramping(self, now: float, is_overload: bool): + """RAMPING: Increasing current limit""" + if is_overload: + self._handle_overload_event(now) + return + new_limit = self.ramp_controller.update(now) + if new_limit is not None: + self._set_current_limit(new_limit) + if not self.ramp_controller.is_ramping: + self.logger.info("Ramp complete, entering stable state") + self._transition_to(self.STATE_STABLE) + + def _handle_cooldown(self, now: float): + """COOLDOWN: Waiting at initial current after overload""" + elapsed = now - self.state_enter_time + remaining = RAMP_CONFIG['cooldown_duration'] - elapsed + self.dbus_service['/Ramp/TimeRemaining'] = max(0, int(remaining)) + + if elapsed >= RAMP_CONFIG['cooldown_duration']: + self.logger.info("Cooldown complete, starting recovery ramp") + self.ramp_controller.start_ramp(now) + self.overload_detector.set_ramp_start(now) + self._transition_to(self.STATE_RECOVERY) + + def _handle_recovery(self, now: float, is_overload: bool): + """RECOVERY: Ramping back up after overload""" + if is_overload: + self._handle_overload_event(now) + return + + # Check if output power increased - may allow higher target + result = self.ramp_controller.check_output_power_increase(now) + if result: + self.logger.info( + f"Output power increase detected during recovery: " + f"target raised to {result['new_target']}A" + ) + + new_limit = self.ramp_controller.update(now) + if new_limit is not None: + self._set_current_limit(new_limit) + if not self.ramp_controller.is_ramping: + self.logger.info("Recovery complete, entering stable state") + self._transition_to(self.STATE_STABLE) + + def _handle_stable(self, now: float, is_overload: bool): + """STABLE: Maintaining current limit, monitoring for overload""" + if is_overload: + self._handle_overload_event(now) + return + + # Check if output power increased - may allow higher target + result = self.ramp_controller.check_output_power_increase(now) + if result and result.get('should_ramp'): + self.logger.info( + f"Output power increased to {result['output_power']:.0f}W, " + f"raising target to {result['new_target']}A and resuming ramp" + ) + self.ramp_controller.start_ramp(now) + self.overload_detector.set_ramp_start(now) + self._transition_to(self.STATE_RECOVERY) + return + + if self.ramp_controller.check_history_clear(now): + self.logger.info("Overload history cleared after stable operation") + if self.ramp_controller.should_retry_full_power(): + self.logger.info("Attempting full power ramp") + self.ramp_controller.start_ramp(now) + self.overload_detector.set_ramp_start(now) + self._transition_to(self.STATE_RAMPING) + + def _handle_overload_event(self, now: float): + """Handle an overload detection""" + output_power = self.output_l1_power + self.output_l2_power + current_limit = self.current_limit_setting + + # Dump verbose debug info BEFORE resetting the detector + self.overload_detector.dump_overload_debug( + current_limit=current_limit, + output_power=output_power + ) + + result = self.ramp_controller.handle_overload(now, output_power=output_power) + + rapid_info = "" + if result.get('is_rapid_overload'): + rapid_info = f" [RAPID #{result['rapid_overload_count']}]" + + self.logger.warning( + f"Overload #{result['overload_count']}{rapid_info}: " + f"rolling back to {result['new_limit']}A, " + f"recovery target: {result['recovery_target']}A, " + f"fast target: {result['fast_recovery_target']:.1f}A " + f"(output: {output_power:.0f}W)" + ) + self._set_current_limit(result['new_limit']) + self.overload_detector.reset() + self._transition_to(self.STATE_COOLDOWN) + + +def main(): + """Main entry point""" + DBusGMainLoop(set_as_default=True) + + print("=" * 60) + print(f"Generator Current Ramp Controller v{VERSION}") + print("=" * 60) + + mainloop = None + + def signal_handler(signum, frame): + """Handle shutdown signals gracefully""" + sig_name = signal.Signals(signum).name + logging.info(f"Received {sig_name}, shutting down...") + if mainloop is not None: + mainloop.quit() + + # Register signal handlers for graceful shutdown + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + + try: + controller = GeneratorRampController() + mainloop = GLib.MainLoop() + mainloop.run() + except KeyboardInterrupt: + print("\nShutdown requested") + except Exception as e: + logging.error(f"Fatal error: {e}", exc_info=True) + sys.exit(1) + finally: + logging.info("Service stopped") + + +if __name__ == '__main__': + main() diff --git a/dbus-generator-ramp/debug_input_tracker.py b/dbus-generator-ramp/debug_input_tracker.py new file mode 100755 index 0000000..22ea541 --- /dev/null +++ b/dbus-generator-ramp/debug_input_tracker.py @@ -0,0 +1,449 @@ +#!/usr/bin/env python3 +""" +Debug Input Tracker for Generator Overload Detection + +Monitors and outputs all tracked input variables to help debug +generator overload detection. Samples internally at dbus rate (2Hz) +while allowing configurable output frequency. + +Usage: + ./debug_input_tracker.py # Output every 2 seconds + ./debug_input_tracker.py --interval 5 # Output every 5 seconds + ./debug_input_tracker.py --interval 0.5 # Output every sample + ./debug_input_tracker.py --csv # CSV output format + ./debug_input_tracker.py --verbose # Include raw buffer data + +Author: Claude (Anthropic) +License: MIT +""" + +import sys +import os +import argparse +from time import time, sleep +from datetime import datetime +from collections import deque + +# Add velib_python to path (Venus OS standard location) +sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'ext', 'velib_python')) +sys.path.insert(1, '/opt/victronenergy/velib_python') + +try: + import dbus +except ImportError: + print("ERROR: dbus not available. This script must run on Venus OS.") + print("For development testing, use --mock flag.") + dbus = None + +from config import DBUS_CONFIG, OVERLOAD_CONFIG +from overload_detector import OverloadDetector + + +class InputTracker: + """ + Tracks generator input power and deviation metrics for debugging. + """ + + def __init__(self, use_mock=False): + self.use_mock = use_mock + self.bus = None + + if not use_mock: + if dbus is None: + raise RuntimeError("dbus module not available") + self.bus = dbus.SystemBus() + + # Services + self.vebus_service = DBUS_CONFIG['vebus_service'] + self.generator_service = DBUS_CONFIG['generator_service'] + + # Overload detector for deviation tracking + self.detector = OverloadDetector(OVERLOAD_CONFIG) + + # Internal sample buffers for statistics + self.sample_count = 0 + self.power_samples = deque(maxlen=100) # Last 100 samples for stats + + # Track min/max/avg since last output + self.reset_interval_stats() + + # Mock state for testing + self._mock_time = 0 + self._mock_base_power = 8000 + + def reset_interval_stats(self): + """Reset statistics tracked between output intervals""" + self.interval_samples = 0 + self.interval_power_min = float('inf') + self.interval_power_max = float('-inf') + self.interval_power_sum = 0 + self.interval_overload_triggers = 0 + + def _get_dbus_value(self, service, path): + """Get a value from D-Bus service""" + if self.use_mock: + return self._get_mock_value(service, path) + try: + obj = self.bus.get_object(service, path, introspect=False) + return obj.GetValue(dbus_interface='com.victronenergy.BusItem') + except Exception: + return None + + def _get_mock_value(self, service, path): + """Return mock values for development testing""" + import random + import math + + self._mock_time += 0.5 + + if path == '/Ac/ActiveIn/L1/P': + # Simulate power with some oscillation + oscillation = 200 * math.sin(self._mock_time * 0.5) + noise = random.gauss(0, 50) + return self._mock_base_power + oscillation + noise + elif path == '/Ac/ActiveIn/L2/P': + return 0 # Single phase + elif path == '/Ac/ActiveIn/Connected': + return 1 + elif path == '/State': + return 1 # Running + elif path == '/Ac/In/1/CurrentLimit': + return 45.0 + return None + + def read_inputs(self): + """Read all input values from dbus""" + # Power readings + l1_power = self._get_dbus_value(self.vebus_service, '/Ac/ActiveIn/L1/P') + l2_power = self._get_dbus_value(self.vebus_service, '/Ac/ActiveIn/L2/P') + + l1_power = float(l1_power) if l1_power is not None else 0.0 + l2_power = float(l2_power) if l2_power is not None else 0.0 + total_power = l1_power + l2_power + + # AC connection status + ac_connected = self._get_dbus_value(self.vebus_service, '/Ac/ActiveIn/Connected') + ac_connected = bool(ac_connected) if ac_connected is not None else False + + # Generator state + gen_state = self._get_dbus_value(self.generator_service, '/State') + gen_state = int(gen_state) if gen_state is not None else 0 + + # Current limit + current_limit = self._get_dbus_value(self.vebus_service, '/Ac/In/1/CurrentLimit') + current_limit = float(current_limit) if current_limit is not None else 0.0 + + return { + 'timestamp': time(), + 'l1_power': l1_power, + 'l2_power': l2_power, + 'total_power': total_power, + 'ac_connected': ac_connected, + 'generator_state': gen_state, + 'current_limit': current_limit, + } + + def sample(self): + """ + Take a single sample and update internal tracking. + Returns sample data with overload detection diagnostics. + """ + inputs = self.read_inputs() + + # Update overload detector + is_overload, diag = self.detector.update( + inputs['l1_power'], + inputs['l2_power'], + inputs['timestamp'] + ) + + self.sample_count += 1 + self.power_samples.append(inputs['total_power']) + + # Update interval statistics + self.interval_samples += 1 + self.interval_power_sum += inputs['total_power'] + self.interval_power_min = min(self.interval_power_min, inputs['total_power']) + self.interval_power_max = max(self.interval_power_max, inputs['total_power']) + if is_overload: + self.interval_overload_triggers += 1 + + return { + **inputs, + 'is_overload': is_overload, + 'diagnostics': diag, + 'sample_number': self.sample_count, + } + + def get_interval_stats(self): + """Get statistics for the current output interval""" + if self.interval_samples == 0: + return None + + return { + 'samples': self.interval_samples, + 'power_min': self.interval_power_min, + 'power_max': self.interval_power_max, + 'power_avg': self.interval_power_sum / self.interval_samples, + 'power_range': self.interval_power_max - self.interval_power_min, + 'overload_triggers': self.interval_overload_triggers, + } + + def get_buffer_stats(self): + """Get statistics from the power sample buffer""" + if len(self.power_samples) < 2: + return None + + samples = list(self.power_samples) + n = len(samples) + mean = sum(samples) / n + variance = sum((x - mean) ** 2 for x in samples) / n + std_dev = variance ** 0.5 + + return { + 'buffer_size': n, + 'mean': mean, + 'std_dev': std_dev, + 'min': min(samples), + 'max': max(samples), + } + + +def format_table_output(sample, interval_stats, buffer_stats, verbose=False): + """Format output as a readable table""" + lines = [] + + # Header with timestamp + ts = datetime.fromtimestamp(sample['timestamp']).strftime('%Y-%m-%d %H:%M:%S.%f')[:-3] + lines.append(f"\n{'='*70}") + lines.append(f"Debug Input Tracker - {ts}") + lines.append(f"{'='*70}") + + # Generator Status + gen_states = {0: 'Stopped', 1: 'Running', 2: 'Warm-up', 3: 'Cool-down', 10: 'Error'} + gen_state_name = gen_states.get(sample['generator_state'], f"Unknown({sample['generator_state']})") + + lines.append(f"\n--- Generator Status ---") + lines.append(f" State: {gen_state_name}") + lines.append(f" AC Connected: {'Yes' if sample['ac_connected'] else 'No'}") + lines.append(f" Current Limit: {sample['current_limit']:.1f} A") + + # Power Readings + lines.append(f"\n--- Power Readings ---") + lines.append(f" L1 Power: {sample['l1_power']:>8.1f} W") + lines.append(f" L2 Power: {sample['l2_power']:>8.1f} W") + lines.append(f" Total Power: {sample['total_power']:>8.1f} W") + + # Interval Statistics + if interval_stats: + lines.append(f"\n--- Interval Stats ({interval_stats['samples']} samples) ---") + lines.append(f" Power Min: {interval_stats['power_min']:>8.1f} W") + lines.append(f" Power Max: {interval_stats['power_max']:>8.1f} W") + lines.append(f" Power Avg: {interval_stats['power_avg']:>8.1f} W") + lines.append(f" Power Range: {interval_stats['power_range']:>8.1f} W") + + # Deviation/Detection Metrics + diag = sample['diagnostics'] + lines.append(f"\n--- Overload Detection ---") + + if diag.get('status') == 'warming_up': + lines.append(f" Status: Warming up ({diag.get('samples', 0)}/{diag.get('needed', '?')} samples)") + elif diag.get('status') == 'lockout': + lines.append(f" Status: Lockout ({diag.get('lockout_remaining', 0):.1f}s remaining)") + else: + # Method 1: Reversals + reversals = diag.get('reversals', 0) + reversal_thresh = diag.get('reversal_threshold', OVERLOAD_CONFIG['reversal_threshold']) + method1 = diag.get('method1_triggered', False) + lines.append(f" Reversals: {reversals:>3d} / {reversal_thresh} threshold {'[TRIGGERED]' if method1 else ''}") + + # Method 2: Std Dev + std_dev = diag.get('std_dev', 0) + std_thresh = diag.get('std_dev_threshold', OVERLOAD_CONFIG['std_dev_threshold']) + method2 = diag.get('method2_triggered', False) + lines.append(f" Std Deviation: {std_dev:>7.1f} W / {std_thresh} W threshold {'[TRIGGERED]' if method2 else ''}") + + # Combined detection + instant = diag.get('instant_detection', False) + confirmed = diag.get('confirmed_count', 0) + confirm_thresh = diag.get('confirmation_threshold', OVERLOAD_CONFIG['confirmation_threshold']) + lines.append(f" Instant Detect: {'Yes' if instant else 'No'}") + lines.append(f" Confirmations: {confirmed:>3d} / {confirm_thresh} threshold") + + # Final overload status + is_overload = sample['is_overload'] + status = "*** OVERLOAD DETECTED ***" if is_overload else "Normal" + lines.append(f" Status: {status}") + + # Verbose buffer data + if verbose and buffer_stats: + lines.append(f"\n--- Buffer Statistics ({buffer_stats['buffer_size']} samples) ---") + lines.append(f" Mean Power: {buffer_stats['mean']:>8.1f} W") + lines.append(f" Std Deviation: {buffer_stats['std_dev']:>8.1f} W") + lines.append(f" Min: {buffer_stats['min']:>8.1f} W") + lines.append(f" Max: {buffer_stats['max']:>8.1f} W") + + lines.append(f"\n Sample #: {sample['sample_number']}") + + return '\n'.join(lines) + + +def format_csv_output(sample, interval_stats, include_header=False): + """Format output as CSV""" + headers = [ + 'timestamp', 'sample_num', 'gen_state', 'ac_connected', 'current_limit', + 'l1_power', 'l2_power', 'total_power', + 'reversals', 'std_dev', 'method1_triggered', 'method2_triggered', + 'instant_detection', 'confirmed_count', 'is_overload', + 'interval_samples', 'power_min', 'power_max', 'power_avg', 'power_range' + ] + + diag = sample['diagnostics'] + + values = [ + datetime.fromtimestamp(sample['timestamp']).strftime('%Y-%m-%d %H:%M:%S.%f')[:-3], + sample['sample_number'], + sample['generator_state'], + 1 if sample['ac_connected'] else 0, + f"{sample['current_limit']:.1f}", + f"{sample['l1_power']:.1f}", + f"{sample['l2_power']:.1f}", + f"{sample['total_power']:.1f}", + diag.get('reversals', ''), + f"{diag.get('std_dev', 0):.1f}" if 'std_dev' in diag else '', + 1 if diag.get('method1_triggered', False) else 0, + 1 if diag.get('method2_triggered', False) else 0, + 1 if diag.get('instant_detection', False) else 0, + diag.get('confirmed_count', ''), + 1 if sample['is_overload'] else 0, + ] + + # Interval stats + if interval_stats: + values.extend([ + interval_stats['samples'], + f"{interval_stats['power_min']:.1f}", + f"{interval_stats['power_max']:.1f}", + f"{interval_stats['power_avg']:.1f}", + f"{interval_stats['power_range']:.1f}", + ]) + else: + values.extend(['', '', '', '', '']) + + lines = [] + if include_header: + lines.append(','.join(headers)) + lines.append(','.join(str(v) for v in values)) + + return '\n'.join(lines) + + +def main(): + parser = argparse.ArgumentParser( + description='Debug tracker for generator input variables and overload detection', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + %(prog)s # Output every 2 seconds (default) + %(prog)s --interval 5 # Output every 5 seconds + %(prog)s --interval 0.5 # Output every sample (500ms) + %(prog)s --csv # CSV format for logging + %(prog)s --csv > debug.csv # Log to file + %(prog)s --verbose # Include buffer statistics + %(prog)s --mock # Use mock data for testing +""" + ) + + parser.add_argument( + '-i', '--interval', + type=float, + default=2.0, + help='Output interval in seconds (default: 2.0, min: 0.5)' + ) + + parser.add_argument( + '--csv', + action='store_true', + help='Output in CSV format' + ) + + parser.add_argument( + '-v', '--verbose', + action='store_true', + help='Include detailed buffer statistics' + ) + + parser.add_argument( + '--mock', + action='store_true', + help='Use mock data for development testing' + ) + + parser.add_argument( + '--sample-rate', + type=float, + default=0.5, + help='Internal sample rate in seconds (default: 0.5 = 2Hz dbus rate)' + ) + + args = parser.parse_args() + + # Validate interval + if args.interval < args.sample_rate: + args.interval = args.sample_rate + + # Initialize tracker + try: + tracker = InputTracker(use_mock=args.mock) + except RuntimeError as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + + if not args.csv: + print("Starting debug input tracker...") + print(f" Sample rate: {args.sample_rate}s ({1/args.sample_rate:.1f} Hz)") + print(f" Output interval: {args.interval}s") + print(f" Press Ctrl+C to stop\n") + + # Output CSV header + first_output = True + + # Timing + last_output_time = 0 + + try: + while True: + now = time() + + # Take a sample + sample = tracker.sample() + + # Check if it's time to output + if now - last_output_time >= args.interval: + interval_stats = tracker.get_interval_stats() + buffer_stats = tracker.get_buffer_stats() if args.verbose else None + + if args.csv: + print(format_csv_output(sample, interval_stats, include_header=first_output)) + else: + print(format_table_output(sample, interval_stats, buffer_stats, args.verbose)) + + # Flush for real-time output when piping + sys.stdout.flush() + + # Reset interval tracking + tracker.reset_interval_stats() + last_output_time = now + first_output = False + + # Wait for next sample + sleep(args.sample_rate) + + except KeyboardInterrupt: + if not args.csv: + print("\n\nStopped.") + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/dbus-generator-ramp/deploy.sh b/dbus-generator-ramp/deploy.sh new file mode 100755 index 0000000..5fe4b9b --- /dev/null +++ b/dbus-generator-ramp/deploy.sh @@ -0,0 +1,224 @@ +#!/bin/bash +# +# Deploy script for Generator Current Ramp Controller +# +# Deploys a built package to a CerboGX device via SSH. +# +# Usage: +# ./deploy.sh [package.tar.gz] +# ./deploy.sh 192.168.1.100 # Uses latest package in current dir +# ./deploy.sh 192.168.1.100 dbus-generator-ramp-1.0.0.tar.gz +# ./deploy.sh 192.168.1.100 --webui # Also install web UI +# +# Prerequisites: +# - SSH access enabled on CerboGX (Settings > General > SSH) +# - Package built with ./build-package.sh +# + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Default values +CERBO_IP="" +PACKAGE="" +INSTALL_WEBUI="" +SSH_USER="root" +REMOTE_DIR="/data" + +# Print usage +usage() { + echo "Usage: $0 [OPTIONS] [package.tar.gz]" + echo "" + echo "Arguments:" + echo " cerbo-ip IP address of the CerboGX" + echo " package.tar.gz Package file (default: latest in current directory)" + echo "" + echo "Options:" + echo " --webui Also install the web UI service" + echo " --user USER SSH user (default: root)" + echo " -h, --help Show this help message" + echo "" + echo "Examples:" + echo " $0 192.168.1.100" + echo " $0 192.168.1.100 --webui" + echo " $0 192.168.1.100 dbus-generator-ramp-1.2.0.tar.gz" + echo "" + echo "Prerequisites:" + echo " 1. Enable SSH on CerboGX: Settings > General > SSH" + echo " 2. Build package first: ./build-package.sh" + exit 1 +} + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --webui) + INSTALL_WEBUI="--webui" + shift + ;; + --user) + SSH_USER="$2" + shift 2 + ;; + -h|--help) + usage + ;; + -*) + echo -e "${RED}Error: Unknown option: $1${NC}" + usage + ;; + *) + if [ -z "$CERBO_IP" ]; then + CERBO_IP="$1" + elif [ -z "$PACKAGE" ]; then + PACKAGE="$1" + else + echo -e "${RED}Error: Too many arguments${NC}" + usage + fi + shift + ;; + esac +done + +# Check required arguments +if [ -z "$CERBO_IP" ]; then + echo -e "${RED}Error: CerboGX IP address required${NC}" + echo "" + usage +fi + +# Find package if not specified +if [ -z "$PACKAGE" ]; then + # Look for latest package in script directory + PACKAGE=$(ls -t "$SCRIPT_DIR"/dbus-generator-ramp-*.tar.gz 2>/dev/null | head -1) + if [ -z "$PACKAGE" ]; then + echo -e "${RED}Error: No package found. Run ./build-package.sh first${NC}" + exit 1 + fi +fi + +# Verify package exists +if [ ! -f "$PACKAGE" ]; then + # Try in script directory + if [ -f "$SCRIPT_DIR/$PACKAGE" ]; then + PACKAGE="$SCRIPT_DIR/$PACKAGE" + else + echo -e "${RED}Error: Package not found: $PACKAGE${NC}" + exit 1 + fi +fi + +PACKAGE_NAME=$(basename "$PACKAGE") + +echo "==================================================" +echo "Deploying to CerboGX" +echo "==================================================" +echo "Target: $SSH_USER@$CERBO_IP" +echo "Package: $PACKAGE_NAME" +echo "Web UI: ${INSTALL_WEBUI:-no}" +echo "" + +# Test SSH connection +echo -e "${YELLOW}1. Testing SSH connection...${NC}" +if ! ssh -o ConnectTimeout=5 -o BatchMode=yes "$SSH_USER@$CERBO_IP" "echo 'SSH OK'" 2>/dev/null; then + echo -e "${RED} SSH connection failed.${NC}" + echo "" + echo " Troubleshooting:" + echo " - Verify CerboGX IP address: $CERBO_IP" + echo " - Enable SSH: Settings > General > SSH on the CerboGX" + echo " - Check password/key authentication" + echo "" + echo " Try manual connection:" + echo " ssh $SSH_USER@$CERBO_IP" + exit 1 +fi +echo -e "${GREEN} SSH connection OK${NC}" + +# Copy package +echo -e "${YELLOW}2. Copying package to CerboGX...${NC}" +scp "$PACKAGE" "$SSH_USER@$CERBO_IP:$REMOTE_DIR/" +echo -e "${GREEN} Package copied${NC}" + +# Install on CerboGX +echo -e "${YELLOW}3. Installing on CerboGX...${NC}" +ssh "$SSH_USER@$CERBO_IP" bash </dev/null || true + # Wait for service to fully stop and release D-Bus name + sleep 3 +fi + +# Remove old installation directory (but keep learned data) +if [ -d $REMOTE_DIR/dbus-generator-ramp ]; then + echo " Backing up learned model..." + if [ -f $REMOTE_DIR/dbus-generator-ramp/learned_model.json ]; then + cp $REMOTE_DIR/dbus-generator-ramp/learned_model.json /tmp/learned_model.json.bak 2>/dev/null || true + fi + echo " Removing old installation..." + rm -rf $REMOTE_DIR/dbus-generator-ramp +fi + +# Extract new package +echo " Extracting package..." +tar -xzf $PACKAGE_NAME + +# Restore learned model if it existed +if [ -f /tmp/learned_model.json.bak ]; then + echo " Restoring learned model..." + mkdir -p $REMOTE_DIR/dbus-generator-ramp + mv /tmp/learned_model.json.bak $REMOTE_DIR/dbus-generator-ramp/learned_model.json +fi + +# Run install script +echo " Running install.sh..." +cd $REMOTE_DIR/dbus-generator-ramp +./install.sh $INSTALL_WEBUI + +# Clean up package file +rm -f $REMOTE_DIR/$PACKAGE_NAME +EOF + +echo -e "${GREEN} Installation complete${NC}" + +# Check service status +echo -e "${YELLOW}4. Checking service status...${NC}" +sleep 2 +ssh "$SSH_USER@$CERBO_IP" bash <<'EOF' +if command -v svstat >/dev/null 2>&1; then + echo "" + svstat /service/dbus-generator-ramp 2>/dev/null || echo " Service not yet supervised" + echo "" + echo "Recent logs:" + if [ -f /var/log/dbus-generator-ramp/current ]; then + cat /var/log/dbus-generator-ramp/current | tai64nlocal 2>/dev/null | tail -10 + else + echo " No logs yet" + fi +fi +EOF + +echo "" +echo "==================================================" +echo -e "${GREEN}Deployment complete!${NC}" +echo "==================================================" +echo "" +echo "Useful commands (on CerboGX):" +echo " svstat /service/dbus-generator-ramp" +echo " tail -F /var/log/dbus-generator-ramp/current | tai64nlocal" +echo " svc -t /service/dbus-generator-ramp # restart" +echo " svc -d /service/dbus-generator-ramp # stop" +echo "" diff --git a/dbus-generator-ramp/docker-compose.yml b/dbus-generator-ramp/docker-compose.yml new file mode 100644 index 0000000..eee11c5 --- /dev/null +++ b/dbus-generator-ramp/docker-compose.yml @@ -0,0 +1,62 @@ +# docker-compose.yml for Generator Current Ramp Controller Development +# +# Usage: +# docker-compose build # Build the development image +# docker-compose run --rm dev bash # Interactive shell +# docker-compose run --rm dev python overload_detector.py # Run detector tests +# docker-compose run --rm dev python ramp_controller.py # Run ramp tests +# +# Note: Full D-Bus integration requires a real Venus OS device. +# This environment is for unit testing components. + +version: '3.8' + +services: + dev: + build: + context: . + dockerfile: Dockerfile + + volumes: + # Mount source code for live editing + - .:/app + + # Mount test data if you have recorded samples + # - ./test_data:/app/test_data + + environment: + - PYTHONPATH=/app:/app/ext/velib_python + - PYTHONUNBUFFERED=1 + + # Keep container running for interactive use + stdin_open: true + tty: true + + # Default command + command: bash + + # Optional: Mock D-Bus service for integration testing + # This would require additional setup to mock Venus OS services + mock-dbus: + build: + context: . + dockerfile: Dockerfile + + volumes: + - .:/app + - dbus-socket:/var/run/dbus + + environment: + - DBUS_SYSTEM_BUS_ADDRESS=unix:path=/var/run/dbus/system_bus_socket + + command: > + bash -c " + dbus-daemon --system --fork && + echo 'Mock D-Bus running' && + tail -f /dev/null + " + + privileged: true + +volumes: + dbus-socket: diff --git a/dbus-generator-ramp/install.sh b/dbus-generator-ramp/install.sh new file mode 100755 index 0000000..c9e5da0 --- /dev/null +++ b/dbus-generator-ramp/install.sh @@ -0,0 +1,233 @@ +#!/bin/bash +# +# Installation script for Generator Current Ramp Controller +# +# Run this on the Venus OS device after copying files to /data/dbus-generator-ramp/ +# +# Usage: +# chmod +x install.sh +# ./install.sh # Install main service only +# ./install.sh --webui # Install main service + web UI +# + +set -e + +INSTALL_DIR="/data/dbus-generator-ramp" +INSTALL_WEBUI=false + +# Find velib_python - check standard location first, then search existing services +VELIB_DIR="" +if [ -d "/opt/victronenergy/velib_python" ]; then + VELIB_DIR="/opt/victronenergy/velib_python" +else + # Search for velib_python in existing Venus OS services (in order of preference) + for candidate in \ + "/opt/victronenergy/dbus-systemcalc-py/ext/velib_python" \ + "/opt/victronenergy/dbus-generator/ext/velib_python" \ + "/opt/victronenergy/dbus-mqtt/ext/velib_python" \ + "/opt/victronenergy/dbus-digitalinputs/ext/velib_python" \ + "/opt/victronenergy/vrmlogger/ext/velib_python" + do + if [ -d "$candidate" ] && [ -f "$candidate/vedbus.py" ]; then + VELIB_DIR="$candidate" + break + fi + done +fi + +if [ -z "$VELIB_DIR" ]; then + # Last resort: find any vedbus.py + VEDBUS_PATH=$(find /opt/victronenergy -name "vedbus.py" -path "*/velib_python/*" 2>/dev/null | head -1) + if [ -n "$VEDBUS_PATH" ]; then + VELIB_DIR=$(dirname "$VEDBUS_PATH") + fi +fi + +# Determine the correct service directory +# Venus OS uses either /service or /opt/victronenergy/service depending on version +if [ -d "/service" ] && [ ! -L "/service" ]; then + # /service is a real directory (some Venus OS versions) + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +elif [ -L "/service" ]; then + # /service is a symlink, follow it + SERVICE_DIR=$(readlink -f /service) +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +# Parse arguments +if [ "$1" = "--webui" ]; then + INSTALL_WEBUI=true +fi + +echo "==================================================" +echo "Generator Current Ramp Controller - Installation" +echo "==================================================" + +# Check if running on Venus OS +if [ ! -d "$SERVICE_DIR" ]; then + echo "ERROR: This doesn't appear to be a Venus OS device." + echo " Service directory not found." + echo " Checked: /service, /opt/victronenergy/service" + exit 1 +fi + +echo "Detected service directory: $SERVICE_DIR" + +# Check if files exist +if [ ! -f "$INSTALL_DIR/dbus-generator-ramp.py" ]; then + echo "ERROR: Installation files not found in $INSTALL_DIR" + echo " Please copy all files to $INSTALL_DIR first." + exit 1 +fi + +echo "1. Making scripts executable..." +chmod +x "$INSTALL_DIR/service/run" +chmod +x "$INSTALL_DIR/service/log/run" +chmod +x "$INSTALL_DIR/dbus-generator-ramp.py" +if [ -f "$INSTALL_DIR/service-webui/run" ]; then + chmod +x "$INSTALL_DIR/service-webui/run" + chmod +x "$INSTALL_DIR/service-webui/log/run" + chmod +x "$INSTALL_DIR/web_ui.py" +fi + +echo "2. Creating velib_python symlink..." +if [ -z "$VELIB_DIR" ]; then + echo "ERROR: Could not find velib_python on this system." + echo " Searched /opt/victronenergy for vedbus.py" + exit 1 +fi +echo " Found velib_python at: $VELIB_DIR" +mkdir -p "$INSTALL_DIR/ext" +# Remove existing symlink if it points somewhere else +if [ -L "$INSTALL_DIR/ext/velib_python" ]; then + CURRENT_TARGET=$(readlink "$INSTALL_DIR/ext/velib_python") + if [ "$CURRENT_TARGET" != "$VELIB_DIR" ]; then + echo " Updating symlink (was: $CURRENT_TARGET)" + rm "$INSTALL_DIR/ext/velib_python" + fi +fi +if [ ! -L "$INSTALL_DIR/ext/velib_python" ]; then + ln -s "$VELIB_DIR" "$INSTALL_DIR/ext/velib_python" + echo " Symlink created: $INSTALL_DIR/ext/velib_python -> $VELIB_DIR" +else + echo " Symlink already exists" +fi + +echo "3. Creating main service symlink..." +if [ -L "$SERVICE_DIR/dbus-generator-ramp" ]; then + echo " Service link already exists, removing old link..." + rm "$SERVICE_DIR/dbus-generator-ramp" +fi +if [ -e "$SERVICE_DIR/dbus-generator-ramp" ]; then + echo " Removing existing service directory..." + rm -rf "$SERVICE_DIR/dbus-generator-ramp" +fi +ln -s "$INSTALL_DIR/service" "$SERVICE_DIR/dbus-generator-ramp" + +# Verify symlink was created +if [ -L "$SERVICE_DIR/dbus-generator-ramp" ]; then + echo " Symlink created: $SERVICE_DIR/dbus-generator-ramp -> $INSTALL_DIR/service" +else + echo "ERROR: Failed to create service symlink" + exit 1 +fi + +echo "4. Creating log directory..." +mkdir -p /var/log/dbus-generator-ramp + +# Install Web UI if requested +if [ "$INSTALL_WEBUI" = true ]; then + echo "5. Installing Web UI service..." + if [ -L "$SERVICE_DIR/dbus-generator-ramp-webui" ]; then + rm "$SERVICE_DIR/dbus-generator-ramp-webui" + fi + ln -s "$INSTALL_DIR/service-webui" "$SERVICE_DIR/dbus-generator-ramp-webui" + mkdir -p /var/log/dbus-generator-ramp-webui + echo " Web UI will be available at http://:8088" +fi + +echo "6. Setting up rc.local for persistence..." +RC_LOCAL="/data/rc.local" +if [ ! -f "$RC_LOCAL" ]; then + echo "#!/bin/bash" > "$RC_LOCAL" + chmod +x "$RC_LOCAL" +fi + +# Check if already in rc.local +if ! grep -q "dbus-generator-ramp" "$RC_LOCAL"; then + echo "" >> "$RC_LOCAL" + echo "# Generator Current Ramp Controller" >> "$RC_LOCAL" + echo "if [ ! -L $SERVICE_DIR/dbus-generator-ramp ]; then" >> "$RC_LOCAL" + echo " ln -s /data/dbus-generator-ramp/service $SERVICE_DIR/dbus-generator-ramp" >> "$RC_LOCAL" + echo "fi" >> "$RC_LOCAL" + if [ "$INSTALL_WEBUI" = true ]; then + echo "if [ ! -L $SERVICE_DIR/dbus-generator-ramp-webui ]; then" >> "$RC_LOCAL" + echo " ln -s /data/dbus-generator-ramp/service-webui $SERVICE_DIR/dbus-generator-ramp-webui" >> "$RC_LOCAL" + echo "fi" >> "$RC_LOCAL" + fi + echo " Added to rc.local for persistence across firmware updates" +else + echo " Already in rc.local" +fi + +echo "7. Activating service..." +# Give svscan a moment to detect the new service +sleep 2 +# Check if service is being supervised +if command -v svstat >/dev/null 2>&1; then + if svstat "$SERVICE_DIR/dbus-generator-ramp" 2>/dev/null | grep -q "up"; then + echo " Service is running" + else + echo " Waiting for service to start..." + sleep 3 + fi +fi + +echo "" +echo "==================================================" +echo "Installation complete!" +echo "==================================================" +echo "" +echo "Service directory: $SERVICE_DIR" +echo "" + +# Show current status +if command -v svstat >/dev/null 2>&1; then + echo "Current status:" + svstat "$SERVICE_DIR/dbus-generator-ramp" 2>/dev/null || echo " Service not yet detected by svscan" + echo "" +fi + +echo "To check status:" +echo " svstat $SERVICE_DIR/dbus-generator-ramp" +echo "" +echo "To view logs:" +echo " tail -F /var/log/dbus-generator-ramp/current | tai64nlocal" +echo "" +if [ "$INSTALL_WEBUI" = true ]; then + echo "Web UI:" + echo " http://:8088" + echo " svstat $SERVICE_DIR/dbus-generator-ramp-webui" + echo "" +fi +echo "MQTT Paths:" +echo " N//generatorramp/0/State" +echo " N//generatorramp/0/CurrentLimit" +echo " N//generatorramp/0/Settings/..." +echo "" +echo "To stop the service:" +echo " svc -d $SERVICE_DIR/dbus-generator-ramp" +echo "" +echo "IMPORTANT: The VE.Bus service is set to: com.victronenergy.vebus.ttyS4" +echo " If this is incorrect, edit: $INSTALL_DIR/config.py" +echo "" +echo "Optional: Install native GUI integration:" +echo " ./install_gui.sh" +echo "" +echo "Troubleshooting:" +echo " ls -la $SERVICE_DIR/dbus-generator-ramp" +echo " cat /var/log/dbus-generator-ramp/current | tai64nlocal | tail -20" +echo "" diff --git a/dbus-generator-ramp/install_gui.sh b/dbus-generator-ramp/install_gui.sh new file mode 100755 index 0000000..83bf17f --- /dev/null +++ b/dbus-generator-ramp/install_gui.sh @@ -0,0 +1,224 @@ +#!/bin/bash +# +# Install GUI modifications for Generator Ramp Controller +# +# This script: +# 1. Copies PageSettingsGeneratorRamp.qml to the GUI directory +# 2. Patches PageSettingsGenerator.qml to add our menu item +# 3. Restarts the GUI +# +# Note: These changes will be lost on firmware update. +# Run this script again after updating firmware. +# +# Usage: +# ./install_gui.sh # Install GUI modifications +# ./install_gui.sh --remove # Remove GUI modifications +# + +set -e + +QML_DIR="/opt/victronenergy/gui/qml" +INSTALL_DIR="/data/dbus-generator-ramp" +BACKUP_DIR="/data/dbus-generator-ramp/backup" + +# The menu entry to add to PageSettingsGenerator.qml +MENU_ENTRY=' + MbSubMenu { + description: qsTr("Dynamic current ramp") + subpage: + Component { + PageSettingsGeneratorRamp { + title: qsTr("Dynamic current ramp") + } + } + }' + +# Marker to identify our modification +MARKER="" + +install_gui() { + echo "==================================================" + echo "Installing GUI modifications..." + echo "==================================================" + + # Check if our service is installed + if [ ! -f "$INSTALL_DIR/dbus-generator-ramp.py" ]; then + echo "ERROR: Main service not installed. Run install.sh first." + exit 1 + fi + + # Check if QML directory exists + if [ ! -d "$QML_DIR" ]; then + echo "ERROR: GUI directory not found: $QML_DIR" + exit 1 + fi + + # Create backup directory + mkdir -p "$BACKUP_DIR" + + # Step 1: Copy our QML file + echo "1. Installing PageSettingsGeneratorRamp.qml..." + if [ -f "$INSTALL_DIR/qml/PageSettingsGeneratorRamp.qml" ]; then + cp "$INSTALL_DIR/qml/PageSettingsGeneratorRamp.qml" "$QML_DIR/" + echo " Copied to $QML_DIR/" + else + echo "ERROR: PageSettingsGeneratorRamp.qml not found" + exit 1 + fi + + # Step 2: Backup and patch PageSettingsGenerator.qml + echo "2. Patching PageSettingsGenerator.qml..." + + GENERATOR_QML="$QML_DIR/PageSettingsGenerator.qml" + + # Check if already patched + if grep -q "PageSettingsGeneratorRamp" "$GENERATOR_QML"; then + echo " Already patched, skipping..." + else + # Create backup + if [ ! -f "$BACKUP_DIR/PageSettingsGenerator.qml.orig" ]; then + cp "$GENERATOR_QML" "$BACKUP_DIR/PageSettingsGenerator.qml.orig" + echo " Backup saved to $BACKUP_DIR/" + fi + + # Find the line with "Warm-up & cool-down" MbSubMenu and insert our menu after it + # We'll insert after the closing brace of that MbSubMenu block + + # Create a Python script to do the patching (more reliable than sed for multi-line) + python3 << 'PYTHON_SCRIPT' +import re + +qml_file = "/opt/victronenergy/gui/qml/PageSettingsGenerator.qml" + +with open(qml_file, 'r') as f: + content = f.read() + +# Check if already patched +if 'PageSettingsGeneratorRamp' in content: + print("Already patched") + exit(0) + +# Find the "Warm-up & cool-down" MbSubMenu block and insert after it +# We look for the pattern and insert our menu item after the closing brace + +menu_entry = ''' + MbSubMenu { + description: qsTr("Dynamic current ramp") + subpage: + Component { + PageSettingsGeneratorRamp { + title: qsTr("Dynamic current ramp") + } + } + }''' + +# Find the warm-up submenu and add our entry after it +# The warm-up submenu ends with a closing brace followed by the Detect generator switch +pattern = r'(PageSettingsGeneratorWarmup \{[^}]*\}[^}]*\}[^}]*\})' + +def insert_menu(match): + return match.group(1) + menu_entry + +new_content = re.sub(pattern, insert_menu, content, count=1) + +if new_content == content: + # Alternative: insert before the "Detect generator at AC input" switch + pattern2 = r"(\n\t\tMbSwitch \{\n\t\t\tproperty bool generatorIsSet)" + new_content = re.sub(pattern2, menu_entry + r'\1', content, count=1) + +if new_content == content: + print("ERROR: Could not find insertion point") + exit(1) + +with open(qml_file, 'w') as f: + f.write(new_content) + +print("Patched successfully") +PYTHON_SCRIPT + + echo " Patch applied" + fi + + # Step 3: Add to rc.local for persistence + echo "3. Setting up persistence in rc.local..." + RC_LOCAL="/data/rc.local" + if ! grep -q "install_gui.sh" "$RC_LOCAL" 2>/dev/null; then + echo "" >> "$RC_LOCAL" + echo "# Restore Generator Ramp GUI modifications after firmware update" >> "$RC_LOCAL" + echo "if [ -f /data/dbus-generator-ramp/install_gui.sh ]; then" >> "$RC_LOCAL" + echo " /data/dbus-generator-ramp/install_gui.sh 2>/dev/null &" >> "$RC_LOCAL" + echo "fi" >> "$RC_LOCAL" + echo " Added to rc.local for automatic restoration" + else + echo " Already in rc.local" + fi + + # Step 4: Restart GUI + echo "4. Restarting GUI..." + if [ -d /service/start-gui ]; then + svc -t /service/start-gui + sleep 2 + elif [ -d /service/gui ]; then + svc -t /service/gui + sleep 2 + else + echo " Note: GUI service not found. Restart manually or reboot." + fi + + echo "" + echo "==================================================" + echo "GUI installation complete!" + echo "==================================================" + echo "" + echo "The 'Dynamic current ramp' option should now appear in:" + echo " Settings -> Generator start/stop -> Settings -> Dynamic current ramp" + echo "" + echo "Note: Run this script again after firmware updates." + echo "" +} + +remove_gui() { + echo "==================================================" + echo "Removing GUI modifications..." + echo "==================================================" + + # Remove our QML file + if [ -f "$QML_DIR/PageSettingsGeneratorRamp.qml" ]; then + rm "$QML_DIR/PageSettingsGeneratorRamp.qml" + echo "1. Removed PageSettingsGeneratorRamp.qml" + fi + + # Restore original PageSettingsGenerator.qml + if [ -f "$BACKUP_DIR/PageSettingsGenerator.qml.orig" ]; then + cp "$BACKUP_DIR/PageSettingsGenerator.qml.orig" "$QML_DIR/PageSettingsGenerator.qml" + echo "2. Restored original PageSettingsGenerator.qml" + else + echo "2. No backup found, manual cleanup may be needed" + fi + + # Restart GUI + echo "3. Restarting GUI..." + if [ -d /service/start-gui ]; then + svc -t /service/start-gui + sleep 2 + elif [ -d /service/gui ]; then + svc -t /service/gui + sleep 2 + else + echo " Note: GUI service not found. Restart manually or reboot." + fi + + echo "" + echo "GUI modifications removed." + echo "" +} + +# Main +case "$1" in + --remove) + remove_gui + ;; + *) + install_gui + ;; +esac diff --git a/dbus-generator-ramp/overload_detector.py b/dbus-generator-ramp/overload_detector.py new file mode 100644 index 0000000..b0a95cb --- /dev/null +++ b/dbus-generator-ramp/overload_detector.py @@ -0,0 +1,544 @@ +""" +Overload Detector for Generator Current Ramp Controller + +Detects generator overload by analyzing power fluctuation patterns. +Uses a hybrid approach combining multiple detection methods. + +Key Design Goals: +- Detect erratic oscillations (overload) - TRIGGER +- Ignore smooth load changes (normal) - IGNORE +- Minimize false positives with confirmation buffer +""" + +import logging +from collections import deque +from time import time + +try: + import numpy as np + HAS_NUMPY = True +except ImportError: + HAS_NUMPY = False + import math + +from config import OVERLOAD_CONFIG + + +class OverloadDetector: + """ + Hybrid overload detector combining: + 1. Rate-of-change reversal detection (catches rapid oscillations) + 2. Detrended standard deviation (ignores smooth ramps) + + Both methods must agree, plus confirmation over multiple samples. + """ + + def __init__(self, config=None): + self.config = config or OVERLOAD_CONFIG + self.logger = logging.getLogger('OverloadDetector') + + # Power sample buffer for std dev calculation (INPUT power - actual generator load) + self.power_buffer = deque(maxlen=self.config['std_dev_window_size']) + + # Derivative buffer for reversal detection + self.derivative_buffer = deque(maxlen=self.config['reversal_window_size']) + + # Confirmation buffer + self.confirmation_buffer = deque(maxlen=self.config['confirmation_window']) + + # State tracking + self.last_power = None + self.last_derivative = None + self.lockout_until = 0 + + # Grace period tracking (suppresses detection after ramp starts) + self.ramp_start_time = None + + # Optional smoothing + self.smoothed_power = None + + # Diagnostics + self.last_diagnostics = {} + + def set_ramp_start(self, timestamp: float = None): + """ + Mark the start of a ramp phase. Suppresses detection during grace period. + + Args: + timestamp: Time when ramp started (defaults to time.time()) + """ + if timestamp is None: + timestamp = time() + self.ramp_start_time = timestamp + grace_period = self.config.get('ramp_start_grace_period', 30) + self.logger.info(f"Ramp start marked - grace period: {grace_period}s") + + def update(self, l1_power: float, l2_power: float, timestamp: float = None, + output_power: float = None) -> tuple: + """ + Process new power readings. + + Args: + l1_power: Power on L1 in Watts + l2_power: Power on L2 in Watts (0 if single phase) + timestamp: Current time (defaults to time.time()) + output_power: Total AC output power (for filtering false positives) + + Returns: + tuple: (is_overload: bool, diagnostics: dict) + """ + if timestamp is None: + timestamp = time() + + total_power = (l1_power or 0) + (l2_power or 0) + + # Check lockout + if timestamp < self.lockout_until: + return False, { + 'status': 'lockout', + 'lockout_remaining': self.lockout_until - timestamp + } + + # Check grace period after ramp start + grace_period = self.config.get('ramp_start_grace_period', 30) + if self.ramp_start_time is not None: + elapsed_since_ramp = timestamp - self.ramp_start_time + if elapsed_since_ramp < grace_period: + return False, { + 'status': 'grace_period', + 'grace_remaining': grace_period - elapsed_since_ramp + } + + # Optional smoothing + if self.config['use_smoothing']: + if self.smoothed_power is None: + self.smoothed_power = total_power + else: + alpha = self.config['smoothing_alpha'] + self.smoothed_power = alpha * total_power + (1 - alpha) * self.smoothed_power + analysis_power = self.smoothed_power + else: + analysis_power = total_power + + # Update power buffer (INPUT power - actual generator load) + self.power_buffer.append(analysis_power) + + # Calculate derivative + if self.last_power is not None: + derivative = analysis_power - self.last_power + self.derivative_buffer.append(derivative) + else: + derivative = 0 + + self.last_power = analysis_power + + # Need minimum samples before detection + min_samples = max( + self.config['reversal_window_size'] // 2, + self.config['std_dev_window_size'] // 2 + ) + + if len(self.power_buffer) < min_samples: + return False, { + 'status': 'warming_up', + 'samples': len(self.power_buffer), + 'needed': min_samples + } + + # --- Method 1: Count derivative reversals --- + reversals = self._count_reversals() + method1_triggered = reversals >= self.config['reversal_threshold'] + + # --- Method 2: Detrended standard deviation --- + std_dev = self._detrended_std_dev() + method2_triggered = std_dev > self.config['std_dev_threshold'] + + # --- Combine methods: both must agree --- + instant_detection = method1_triggered and method2_triggered + + # --- Check minimum output power requirement --- + # A true overload requires significant load - without it, fluctuations + # are likely just load switching (e.g., appliance turning off) + min_output = self.config.get('min_output_power_for_overload', 0) + output_power_ok = output_power is None or output_power >= min_output + instant_detection = instant_detection and output_power_ok + + # --- Check trend direction --- + # If power is dropping significantly, it's a load decrease, not overload + # True overloads oscillate at/near a ceiling, not trend downward + trend = self._calculate_trend() + trend_threshold = self.config.get('trend_drop_threshold', -100) + trend_ok = trend >= trend_threshold # OK if trend is not dropping fast + instant_detection = instant_detection and trend_ok + + # --- Check power drop from recent maximum --- + # If current power is well below recent max, we've DROPPED from ceiling + # True overloads oscillate AT/NEAR the ceiling, not below it + # Note: This checks INPUT power - the actual generator load + # Output power is NOT reliable because charger can take over freed capacity + recent_max = max(self.power_buffer) if self.power_buffer else total_power + power_drop = recent_max - total_power + max_drop_allowed = self.config.get('max_power_drop_for_overload', 1000) + power_drop_ok = power_drop <= max_drop_allowed + instant_detection = instant_detection and power_drop_ok + + # --- Confirmation: sustained detection --- + self.confirmation_buffer.append(1 if instant_detection else 0) + confirmed_count = sum(self.confirmation_buffer) + is_overload = confirmed_count >= self.config['confirmation_threshold'] + + # Build diagnostics + self.last_diagnostics = { + 'status': 'monitoring', + 'total_power': total_power, + 'output_power': output_power, + 'min_output_required': min_output, + 'output_power_ok': output_power_ok, + 'trend': round(trend, 1), + 'trend_threshold': trend_threshold, + 'trend_ok': trend_ok, + 'recent_max': round(recent_max, 0), + 'power_drop': round(power_drop, 0), + 'max_drop_allowed': max_drop_allowed, + 'power_drop_ok': power_drop_ok, + 'reversals': reversals, + 'reversal_threshold': self.config['reversal_threshold'], + 'method1_triggered': method1_triggered, + 'std_dev': round(std_dev, 1), + 'std_dev_threshold': self.config['std_dev_threshold'], + 'method2_triggered': method2_triggered, + 'instant_detection': instant_detection, + 'confirmed_count': confirmed_count, + 'confirmation_threshold': self.config['confirmation_threshold'], + 'is_overload': is_overload, + } + + # Apply lockout if triggered + if is_overload: + self.lockout_until = timestamp + self.config['lockout_duration'] + self.logger.warning( + f"OVERLOAD DETECTED! reversals={reversals}, std_dev={std_dev:.1f}W, " + f"trend={trend:.1f}W/s, power_drop={power_drop:.0f}W" + ) + # Clear confirmation buffer after trigger + self.confirmation_buffer.clear() + + return is_overload, self.last_diagnostics + + def _count_reversals(self) -> int: + """ + Count significant sign changes in the derivative. + + A reversal is when the derivative changes sign AND both the + previous and current derivative exceed the threshold magnitude. + """ + if len(self.derivative_buffer) < 3: + return 0 + + threshold = self.config['derivative_threshold'] + derivs = list(self.derivative_buffer) + + reversals = 0 + prev_sign = None + + for d in derivs: + # Ignore small changes + if abs(d) < threshold: + continue + + current_sign = 1 if d > 0 else -1 + + if prev_sign is not None and current_sign != prev_sign: + reversals += 1 + + prev_sign = current_sign + + return reversals + + def _detrended_std_dev(self) -> float: + """ + Calculate standard deviation after removing linear trend. + + This allows us to ignore smooth ramps up or down while + still detecting oscillations around the trend. + """ + if len(self.power_buffer) < 10: + return 0.0 + + data = list(self.power_buffer) + n = len(data) + + if HAS_NUMPY: + # Use numpy for efficiency + arr = np.array(data) + x = np.arange(n) + + # Fit linear trend + coeffs = np.polyfit(x, arr, 1) + trend = np.polyval(coeffs, x) + + # Detrend and calculate std dev + detrended = arr - trend + return float(np.std(detrended)) + else: + # Pure Python fallback + # Calculate linear regression manually + x_mean = (n - 1) / 2.0 + y_mean = sum(data) / n + + numerator = sum((i - x_mean) * (data[i] - y_mean) for i in range(n)) + denominator = sum((i - x_mean) ** 2 for i in range(n)) + + if denominator == 0: + slope = 0 + else: + slope = numerator / denominator + intercept = y_mean - slope * x_mean + + # Calculate detrended values + detrended = [data[i] - (slope * i + intercept) for i in range(n)] + + # Calculate std dev + mean_detrended = sum(detrended) / n + variance = sum((x - mean_detrended) ** 2 for x in detrended) / n + return math.sqrt(variance) + + def _calculate_trend(self) -> float: + """ + Calculate the trend slope in watts per second. + + Returns positive for rising power, negative for falling power. + This helps filter out load-drop scenarios from true overloads. + """ + if len(self.power_buffer) < 10: + return 0.0 + + data = list(self.power_buffer) + n = len(data) + + # Sample interval in seconds + sample_interval = self.config['sample_interval_ms'] / 1000.0 + + if HAS_NUMPY: + arr = np.array(data) + x = np.arange(n) + coeffs = np.polyfit(x, arr, 1) + slope_per_sample = coeffs[0] + else: + x_mean = (n - 1) / 2.0 + y_mean = sum(data) / n + + numerator = sum((i - x_mean) * (data[i] - y_mean) for i in range(n)) + denominator = sum((i - x_mean) ** 2 for i in range(n)) + + if denominator == 0: + slope_per_sample = 0 + else: + slope_per_sample = numerator / denominator + + # Convert from watts/sample to watts/second + return slope_per_sample / sample_interval + + def reset(self): + """Reset detector state (e.g., after generator stops)""" + self.power_buffer.clear() + self.derivative_buffer.clear() + self.confirmation_buffer.clear() + self.last_power = None + self.last_derivative = None + self.smoothed_power = None + self.lockout_until = 0 + self.ramp_start_time = None + self.last_diagnostics = {} + self.logger.debug("Detector reset") + + def get_diagnostics(self) -> dict: + """Get the most recent diagnostics""" + return self.last_diagnostics + + def dump_overload_debug(self, current_limit: float = None, output_power: float = None) -> str: + """ + Generate comprehensive debug dump when overload is detected. + + Call this method when an overload triggers to log detailed data + for algorithm tuning. Returns formatted string and logs it. + + Args: + current_limit: Current input current limit setting (Amps) + output_power: Total AC output power (Watts) + + Returns: + Formatted debug string + """ + diag = self.last_diagnostics + + # Get buffer contents + power_samples = list(self.power_buffer) + deriv_samples = list(self.derivative_buffer) + confirm_samples = list(self.confirmation_buffer) + + # Calculate some additional stats + if power_samples: + power_min = min(power_samples) + power_max = max(power_samples) + power_range = power_max - power_min + power_mean = sum(power_samples) / len(power_samples) + else: + power_min = power_max = power_range = power_mean = 0 + + if deriv_samples: + deriv_min = min(deriv_samples) + deriv_max = max(deriv_samples) + deriv_mean = sum(deriv_samples) / len(deriv_samples) + # Count positive/negative derivatives + deriv_pos = sum(1 for d in deriv_samples if d > self.config['derivative_threshold']) + deriv_neg = sum(1 for d in deriv_samples if d < -self.config['derivative_threshold']) + else: + deriv_min = deriv_max = deriv_mean = deriv_pos = deriv_neg = 0 + + # Format the dump + lines = [ + "", + "=" * 70, + "OVERLOAD DEBUG DUMP", + "=" * 70, + "", + "--- DETECTION RESULT ---", + f" Overload Confirmed: {diag.get('is_overload', False)}", + f" Current Limit: {current_limit}A" if current_limit else " Current Limit: N/A", + f" Output Power: {output_power:.0f}W" if output_power else " Output Power: N/A", + "", + "--- THRESHOLDS (config) ---", + f" reversal_threshold: {self.config['reversal_threshold']}", + f" derivative_threshold: {self.config['derivative_threshold']}W", + f" std_dev_threshold: {self.config['std_dev_threshold']}W", + f" confirmation_threshold: {self.config['confirmation_threshold']}/{self.config['confirmation_window']}", + f" trend_drop_threshold: {self.config.get('trend_drop_threshold', -100)}W/s", + f" max_power_drop_for_overload: {self.config.get('max_power_drop_for_overload', 1000)}W", + f" min_output_power_for_overload: {self.config.get('min_output_power_for_overload', 0)}W", + "", + "--- METHOD 1: REVERSALS ---", + f" Reversals counted: {diag.get('reversals', 0)} (threshold: {self.config['reversal_threshold']})", + f" Method1 triggered: {diag.get('method1_triggered', False)}", + f" Significant positive derivatives: {deriv_pos}", + f" Significant negative derivatives: {deriv_neg}", + f" Derivative range: {deriv_min:.0f}W to {deriv_max:.0f}W", + "", + "--- METHOD 2: STD DEV ---", + f" Detrended std_dev: {diag.get('std_dev', 0):.1f}W (threshold: {self.config['std_dev_threshold']}W)", + f" Method2 triggered: {diag.get('method2_triggered', False)}", + f" Power range: {power_min:.0f}W to {power_max:.0f}W (delta: {power_range:.0f}W)", + f" Power mean: {power_mean:.0f}W", + "", + "--- FILTERS ---", + f" Trend: {diag.get('trend', 0):.1f}W/s (threshold: {diag.get('trend_threshold', -100)}W/s)", + f" Trend OK: {diag.get('trend_ok', True)}", + f" Recent max power: {diag.get('recent_max', 0):.0f}W", + f" Power drop from max: {diag.get('power_drop', 0):.0f}W (max allowed: {diag.get('max_drop_allowed', 1000)}W)", + f" Power drop OK: {diag.get('power_drop_ok', True)}", + f" Output power OK: {diag.get('output_power_ok', True)}", + "", + "--- CONFIRMATION ---", + f" Instant detection: {diag.get('instant_detection', False)}", + f" Confirmed samples: {diag.get('confirmed_count', 0)}/{self.config['confirmation_threshold']}", + f" Confirmation buffer: {confirm_samples}", + "", + "--- POWER BUFFER (last {0} samples) ---".format(len(power_samples)), + ] + + # Add power samples in rows of 10 + for i in range(0, len(power_samples), 10): + chunk = power_samples[i:i+10] + formatted = [f"{p:.0f}" for p in chunk] + lines.append(f" [{i:2d}-{i+len(chunk)-1:2d}]: {', '.join(formatted)}") + + lines.extend([ + "", + "--- DERIVATIVE BUFFER (last {0} samples) ---".format(len(deriv_samples)), + ]) + + # Add derivative samples in rows of 10 + for i in range(0, len(deriv_samples), 10): + chunk = deriv_samples[i:i+10] + formatted = [f"{d:+.0f}" for d in chunk] + lines.append(f" [{i:2d}-{i+len(chunk)-1:2d}]: {', '.join(formatted)}") + + lines.extend([ + "", + "=" * 70, + "", + ]) + + debug_output = "\n".join(lines) + + # Log it + self.logger.warning(debug_output) + + return debug_output + + +class SimpleThresholdDetector: + """ + Simple alternative detector using just power variance. + Use this if the hybrid approach is too sensitive or complex. + """ + + def __init__(self, variance_threshold=100000, window_size=20): + self.variance_threshold = variance_threshold + self.buffer = deque(maxlen=window_size) + self.logger = logging.getLogger('SimpleDetector') + + def update(self, l1_power: float, l2_power: float, timestamp: float = None) -> tuple: + total_power = (l1_power or 0) + (l2_power or 0) + self.buffer.append(total_power) + + if len(self.buffer) < 10: + return False, {'status': 'warming_up'} + + # Calculate variance + data = list(self.buffer) + mean = sum(data) / len(data) + variance = sum((x - mean) ** 2 for x in data) / len(data) + + is_overload = variance > self.variance_threshold + + return is_overload, { + 'variance': variance, + 'threshold': self.variance_threshold, + 'is_overload': is_overload + } + + def reset(self): + self.buffer.clear() + + +# For testing +if __name__ == '__main__': + import random + + logging.basicConfig(level=logging.DEBUG) + detector = OverloadDetector() + + print("Testing with stable power...") + for i in range(50): + power = 8000 + random.gauss(0, 30) + result, diag = detector.update(power, 0) + if i % 10 == 0: + print(f" Sample {i}: power={power:.0f}, overload={result}") + + print("\nTesting with smooth ramp...") + detector.reset() + for i in range(50): + power = 8000 + i * 50 + random.gauss(0, 30) # Ramping up + result, diag = detector.update(power, 0) + if i % 10 == 0: + print(f" Sample {i}: power={power:.0f}, overload={result}") + + print("\nTesting with oscillation (overload)...") + detector.reset() + for i in range(50): + oscillation = 600 * (1 if i % 4 < 2 else -1) # Square wave oscillation + power = 8000 + oscillation + random.gauss(0, 30) + result, diag = detector.update(power, 0) + if result or i % 10 == 0: + print(f" Sample {i}: power={power:.0f}, overload={result}, diag={diag}") diff --git a/dbus-generator-ramp/qml/._PageSettingsGeneratorRamp.qml b/dbus-generator-ramp/qml/._PageSettingsGeneratorRamp.qml new file mode 100644 index 0000000..57fc954 Binary files /dev/null and b/dbus-generator-ramp/qml/._PageSettingsGeneratorRamp.qml differ diff --git a/dbus-generator-ramp/qml/PageSettingsGeneratorRamp.qml b/dbus-generator-ramp/qml/PageSettingsGeneratorRamp.qml new file mode 100644 index 0000000..c92f493 --- /dev/null +++ b/dbus-generator-ramp/qml/PageSettingsGeneratorRamp.qml @@ -0,0 +1,133 @@ +import QtQuick 2 +import com.victron.velib 1.0 +import "utils.js" as Utils + +MbPage { + id: root + title: qsTr("Dynamic current ramp") + + property string bindPrefix: "com.victronenergy.generatorramp" + + // Status items (read-only) + property VBusItem stateItem: VBusItem { bind: Utils.path(bindPrefix, "/State") } + property VBusItem currentLimitItem: VBusItem { bind: Utils.path(bindPrefix, "/CurrentLimit") } + property VBusItem progressItem: VBusItem { bind: Utils.path(bindPrefix, "/Ramp/Progress") } + property VBusItem overloadCountItem: VBusItem { bind: Utils.path(bindPrefix, "/OverloadCount") } + property VBusItem connectedItem: VBusItem { bind: Utils.path(bindPrefix, "/Connected") } + + // State names for display + function stateName(state) { + var names = ["Idle", "Warm-up", "Ramping", "Cooldown", "Recovery", "Stable"] + return state >= 0 && state < names.length ? names[state] : "Unknown" + } + + model: VisibleItemModel { + + MbItemText { + text: qsTr("Status: %1").arg(connectedItem.valid ? stateName(stateItem.value) : "Not running") + show: true + } + + MbItemText { + text: qsTr("Current limit: %1 A | Progress: %2%").arg( + currentLimitItem.valid ? currentLimitItem.value.toFixed(1) : "--").arg( + progressItem.valid ? progressItem.value : "--") + show: connectedItem.valid + } + + MbItemText { + text: qsTr("Overload events this session: %1").arg( + overloadCountItem.valid ? overloadCountItem.value : "0") + show: connectedItem.valid && overloadCountItem.value > 0 + } + + MbSwitch { + id: enableSwitch + name: qsTr("Enable dynamic current ramp") + bind: Utils.path(bindPrefix, "/Settings/Enabled") + enabled: connectedItem.valid + onClicked: { + if (!connectedItem.valid) { + toast.createToast(qsTr("Generator ramp service is not running. Check installation."), 5000, "icon-info-active") + } + } + } + + MbSpinBox { + description: qsTr("Initial current") + show: enableSwitch.checked + item { + bind: Utils.path(bindPrefix, "/Settings/InitialCurrent") + unit: "A" + decimals: 1 + step: 0.5 + min: 10 + max: 100 + } + } + + MbSpinBox { + description: qsTr("Target current") + show: enableSwitch.checked + item { + bind: Utils.path(bindPrefix, "/Settings/TargetCurrent") + unit: "A" + decimals: 1 + step: 0.5 + min: 10 + max: 100 + } + } + + MbSpinBox { + description: qsTr("Ramp duration") + show: enableSwitch.checked + item { + bind: Utils.path(bindPrefix, "/Settings/RampDuration") + unit: "min" + decimals: 0 + step: 5 + min: 1 + max: 120 + } + } + + MbSpinBox { + description: qsTr("Cooldown after overload") + show: enableSwitch.checked + item { + bind: Utils.path(bindPrefix, "/Settings/CooldownDuration") + unit: "min" + decimals: 0 + step: 1 + min: 1 + max: 30 + } + } + + MbSwitch { + name: qsTr("Return to prior current after overload") + bind: Utils.path(bindPrefix, "/Settings/ReturnToStableAfterOverload") + show: enableSwitch.checked + } + + MbSpinBox { + description: qsTr("Min stable time to return to prior current") + show: enableSwitch.checked + item { + bind: Utils.path(bindPrefix, "/Settings/ReturnToStableMinMinutes") + unit: qsTr("min") + decimals: 0 + step: 5 + min: 5 + max: 120 + } + } + + MbItemText { + text: qsTr("Ramps AC input current from initial to target over the specified duration after generator warm-up completes. Automatically reduces current if generator overload is detected. If enabled and you were stable at current for the set time, recovery returns to that prior current (e.g. after a step load like a heater).") + wrapMode: Text.WordWrap + show: enableSwitch.checked + } + } +} diff --git a/dbus-generator-ramp/ramp_controller.py b/dbus-generator-ramp/ramp_controller.py new file mode 100644 index 0000000..31e7883 --- /dev/null +++ b/dbus-generator-ramp/ramp_controller.py @@ -0,0 +1,934 @@ +""" +Ramp Controller for Generator Current Ramp Controller + +Manages the current limit ramping with: +- Initial ramp from 40A to 50A over 30 minutes +- Fast recovery to near-overload point after overload detection +- Adaptive margins based on rapid overload detection +- Output power correlation learning for smarter limits +""" + +import logging +import json +import os +from time import time +from dataclasses import dataclass, field +from typing import List, Optional, Tuple, Dict + +from config import RAMP_CONFIG, LEARNING_CONFIG + + +@dataclass +class OverloadEvent: + """Record of an overload event""" + timestamp: float + current_at_overload: float + last_stable_current: float + output_power: float = 0.0 # Output power at time of overload + + +@dataclass +class LearningDataPoint: + """A data point for correlation learning""" + timestamp: float + output_power: float # Watts + safe_input_current: float # Amps - highest current that was stable + was_overload: bool # True if this was recorded at overload + + +@dataclass +class RampState: + """Current state of the ramp controller""" + current_limit: float + target_limit: float + ramp_start_time: Optional[float] = None + ramp_start_current: Optional[float] = None # Current limit when ramp started + last_stable_current: float = 40.0 + is_recovery: bool = False + # Fast recovery tracking + fast_ramp_target: Optional[float] = None # Target for fast ramp phase + in_fast_ramp: bool = False + + +class PowerCorrelationModel: + """ + Learns the correlation between output power and maximum safe input current. + + Model: max_input_current = base + slope * output_power + + Key insight: When output loads are high, more input power passes through + directly to loads (not through inverter/battery). When output loads are low, + all input power goes to battery charging, stressing the inverter more. + """ + + def __init__(self, config: Dict = None): + self.config = config or LEARNING_CONFIG + self.logger = logging.getLogger('PowerModel') + + # Model parameters + self.base_current = self.config['initial_base_current'] + self.slope = self.config['initial_slope'] + + # Data points for learning + self.data_points: List[LearningDataPoint] = [] + + # Track overload points specifically + self.overload_points: List[Tuple[float, float]] = [] # (output_power, input_current) + + # Confidence level + self.confidence = 0 + + # Load saved model if exists + self._load_model() + + def _model_path(self) -> str: + """Path to saved model file""" + return '/data/dbus-generator-ramp/learned_model.json' + + def _load_model(self): + """Load saved model parameters""" + try: + path = self._model_path() + if os.path.exists(path): + with open(path, 'r') as f: + data = json.load(f) + self.base_current = data.get('base_current', self.base_current) + self.slope = data.get('slope', self.slope) + self.confidence = data.get('confidence', 0) + self.overload_points = [tuple(p) for p in data.get('overload_points', [])] + self.logger.info( + f"Loaded model: base={self.base_current:.1f}A, " + f"slope={self.slope:.6f}, confidence={self.confidence}" + ) + except Exception as e: + self.logger.warning(f"Could not load model: {e}") + + def _save_model(self): + """Save model parameters to disk""" + try: + path = self._model_path() + os.makedirs(os.path.dirname(path), exist_ok=True) + data = { + 'base_current': self.base_current, + 'slope': self.slope, + 'confidence': self.confidence, + 'overload_points': self.overload_points[-20:], # Keep last 20 + } + with open(path, 'w') as f: + json.dump(data, f, indent=2) + except Exception as e: + self.logger.warning(f"Could not save model: {e}") + + def record_stable_operation(self, output_power: float, input_current: float, + timestamp: float = None): + """ + Record a data point of stable operation. + + Args: + output_power: Current output power in Watts + input_current: Current input current limit in Amps + timestamp: Time of recording + """ + if timestamp is None: + timestamp = time() + + point = LearningDataPoint( + timestamp=timestamp, + output_power=output_power, + safe_input_current=input_current, + was_overload=False + ) + + self.data_points.append(point) + + # Trim old data points + max_points = self.config['max_data_points'] + if len(self.data_points) > max_points: + self.data_points = self.data_points[-max_points:] + + self.confidence = min(self.confidence + 1, max_points) + + def record_overload(self, output_power: float, input_current: float, + timestamp: float = None): + """ + Record an overload event - this is valuable learning data. + + Args: + output_power: Output power at time of overload + input_current: Input current when overload occurred + timestamp: Time of overload + """ + if timestamp is None: + timestamp = time() + + # Record as data point + point = LearningDataPoint( + timestamp=timestamp, + output_power=output_power, + safe_input_current=input_current - 1.0, # Overload means this was too high + was_overload=True + ) + self.data_points.append(point) + + # Track overload point + self.overload_points.append((output_power, input_current)) + if len(self.overload_points) > 20: + self.overload_points = self.overload_points[-20:] + + # Update model based on this overload + self._update_model_from_overload(output_power, input_current) + + self.confidence = min(self.confidence + 2, self.config['max_data_points']) + self._save_model() + + self.logger.info( + f"Recorded overload: output={output_power:.0f}W, input={input_current:.1f}A" + ) + + def _update_model_from_overload(self, output_power: float, overload_current: float): + """ + Update model parameters based on overload event. + Uses gradient descent-like update. + """ + lr = self.config['learning_rate'] + + # Current model prediction + predicted_max = self.predict_max_current(output_power) + + # The actual max should be below overload point + actual_max = overload_current - 1.0 + + # Error + error = predicted_max - actual_max + + if error > 0: # Model was too optimistic + # Reduce base_current more aggressively + self.base_current -= lr * error * 0.8 + + # Adjust slope based on where in the power range this occurred + if output_power > 3000: + # Overload at high output - slope might be too high + self.slope -= lr * 0.0001 + + self.logger.info( + f"Model update: base={self.base_current:.1f}A, slope={self.slope:.6f}" + ) + + # Clamp to reasonable values + # Base is the low-output (0-1500W) achievable limit, typically 45A + # Allow learning to adjust slightly but keep within reasonable bounds + self.base_current = max(40.0, min(48.0, self.base_current)) + self.slope = max(0.0, min(0.003, self.slope)) + + def predict_max_current(self, output_power: float) -> float: + """ + Predict the maximum safe input current for given output power. + + Args: + output_power: Current output power in Watts + + Returns: + Predicted maximum safe input current in Amps + """ + # Base prediction from linear model + predicted = self.base_current + (self.slope * output_power) + + # Apply zone-based adjustments + zones = self.config['power_zones'] + for zone_name, (min_p, max_p, offset) in zones.items(): + if min_p <= output_power < max_p: + predicted += offset + break + + # Check against recent overload points + for (op, oc) in self.overload_points: + if abs(op - output_power) < 500: # Similar output power + # Be conservative - use lower of predicted and (overload - margin) + predicted = min(predicted, oc - 3.0) + + return predicted + + def get_confidence_level(self) -> str: + """Get confidence level as string""" + if self.confidence < 3: + return 'LOW' + elif self.confidence < self.config['min_confidence']: + return 'MEDIUM' + else: + return 'HIGH' + + def should_use_model(self) -> bool: + """Check if model has enough confidence to be used""" + return self.confidence >= self.config['min_confidence'] + + def get_status(self) -> Dict: + """Get model status for monitoring""" + return { + 'base_current': self.base_current, + 'slope': self.slope, + 'confidence': self.confidence, + 'confidence_level': self.get_confidence_level(), + 'data_points': len(self.data_points), + 'overload_points': len(self.overload_points), + } + + +class RampController: + """ + Manages current limit ramping with overload protection. + + Behavior: + - Normal ramp: 40A -> 50A over 30 minutes + - On overload: Return to 40A, wait cooldown, then FAST ramp to near-overload + - Fast recovery: Quickly ramp to (overload_point - 4A), then slow ramp to target + - Rapid overload: If overload again within 2 minutes, use larger margin (6A) + - Learning: Track output power correlation to predict safe limits + """ + + def __init__(self, config=None): + self.config = config or RAMP_CONFIG + self.logger = logging.getLogger('RampController') + + # Current state + self.state = RampState( + current_limit=self.config['initial_current'], + target_limit=self.config['target_current'], + last_stable_current=self.config['initial_current'] + ) + + # Overload history + self.overload_history: List[OverloadEvent] = [] + + # Recovery target (may be lower than target_current after overloads) + self.recovery_target = self.config['target_current'] + + # Timestamp tracking + self.last_update_time = None + self.stable_since = None + + # Fast recovery tracking + self.rapid_overload_count = 0 # Count of rapid successive overloads + self.last_overload_time = None + + # Current output power (updated externally) + self.current_output_power = 0.0 + + # Output power at last overload (for detecting significant increases) + self.output_power_at_last_overload = 0.0 + self.last_target_reevaluation = None + + # Power correlation learning + self.power_model = PowerCorrelationModel(LEARNING_CONFIG) + self.learning_config = LEARNING_CONFIG + + @property + def current_limit(self) -> float: + """Get current limit value""" + return self.state.current_limit + + @property + def is_ramping(self) -> bool: + """Check if currently ramping""" + return self.state.ramp_start_time is not None + + def set_output_power(self, power: float): + """Update current output power reading""" + self.current_output_power = power + + def set_initial_limit(self, current_setting: float) -> float: + """ + Called when generator enters warm-up. + Always sets limit to initial value for a known starting state. + """ + initial = self.config['initial_current'] + + if current_setting != initial: + self.logger.info( + f"Generator warm-up: setting limit from {current_setting}A to {initial}A" + ) + else: + self.logger.info( + f"Generator warm-up: confirming limit at {initial}A" + ) + + self.state.current_limit = initial + return initial + + def start_ramp(self, timestamp: float = None): + """ + Start ramping from current position. + """ + if timestamp is None: + timestamp = time() + + self.state.ramp_start_time = timestamp + self.state.is_recovery = len(self.overload_history) > 0 + + if self.state.is_recovery: + # Recovery after cooldown - we rolled back to initial_current + self.state.ramp_start_current = self.config['initial_current'] + self.state.target_limit = self.recovery_target + + # Calculate fast ramp target + self._calculate_fast_recovery_target() + + if self.state.fast_ramp_target and self.state.fast_ramp_target > self.config['initial_current']: + self.state.in_fast_ramp = True + self.logger.info( + f"Starting FAST recovery: {self.state.ramp_start_current}A -> " + f"{self.state.fast_ramp_target:.1f}A (fast), then -> " + f"{self.recovery_target}A (slow)" + ) + else: + self.state.in_fast_ramp = False + self.logger.info( + f"Starting RECOVERY ramp to {self.recovery_target}A at " + f"{self.config['recovery_ramp_rate']} A/min" + ) + else: + # Initial ramp or retry after stable - start from current position + self.state.ramp_start_current = self.state.current_limit + self.state.target_limit = self.config['target_current'] + self.state.fast_ramp_target = None + self.state.in_fast_ramp = False + self.logger.info( + f"Starting INITIAL ramp from {self.state.ramp_start_current}A to " + f"{self.state.target_limit}A at {self.config['initial_ramp_rate']} A/min" + ) + + self.last_update_time = timestamp + + def _calculate_fast_recovery_target(self): + """ + Calculate the target for fast recovery phase. + + Logic: + - Base: (overload_current - fast_recovery_margin) + - If rapid overload: add extra margin + - Consider power model prediction if confident + """ + if not self.overload_history: + self.state.fast_ramp_target = None + return + + last_overload = self.overload_history[-1] + overload_current = last_overload.current_at_overload + + # Base margin + margin = self.config['fast_recovery_margin'] # Default: 4A + + # Add extra margin for rapid overloads + if self.rapid_overload_count > 0: + extra = self.config['rapid_overload_extra_margin'] * self.rapid_overload_count + margin += extra + self.logger.info( + f"Rapid overload #{self.rapid_overload_count}: using margin of {margin}A" + ) + + # Calculate fast ramp target + fast_target = overload_current - margin + + # Consider power model prediction + if self.power_model.should_use_model(): + predicted_max = self.power_model.predict_max_current(self.current_output_power) + # Use the more conservative of the two + if predicted_max < fast_target: + self.logger.info( + f"Power model suggests max {predicted_max:.1f}A at " + f"{self.current_output_power:.0f}W output" + ) + fast_target = min(fast_target, predicted_max - 1.0) + + # Don't fast-ramp below initial_current + fast_target = max(fast_target, self.config['initial_current'] + 1.0) + + # Don't fast-ramp above recovery target + fast_target = min(fast_target, self.recovery_target - 1.0) + + self.state.fast_ramp_target = round(fast_target * 2) / 2 # Round to 0.5A + + self.logger.info( + f"Fast recovery target: {self.state.fast_ramp_target}A " + f"(overload was at {overload_current}A, margin {margin}A)" + ) + + def update(self, timestamp: float = None) -> Optional[float]: + """ + Calculate and return the new current limit based on elapsed time. + """ + if timestamp is None: + timestamp = time() + + if self.state.ramp_start_time is None: + return None + + elapsed = timestamp - self.state.ramp_start_time + + # Determine current phase and rate + if self.state.in_fast_ramp and self.state.fast_ramp_target: + # Fast ramp phase + rate = self.config['fast_ramp_rate'] # Fast: 5 A/min + phase_target = self.state.fast_ramp_target + elif self.state.is_recovery: + # Slow recovery phase + rate = self.config['recovery_ramp_rate'] + phase_target = self.state.target_limit + else: + # Initial ramp phase + rate = self.config['initial_ramp_rate'] + phase_target = self.state.target_limit + + # Calculate new limit from ramp start position + start_current = self.state.ramp_start_current or self.config['initial_current'] + new_limit = start_current + (elapsed * rate / 60.0) + + # Check if we've completed fast ramp phase + if self.state.in_fast_ramp and self.state.fast_ramp_target: + if new_limit >= self.state.fast_ramp_target: + # Transition to slow ramp + self.logger.info( + f"Fast ramp complete at {self.state.fast_ramp_target}A, " + f"switching to slow ramp" + ) + self.state.in_fast_ramp = False + # Reset ramp start time and current for slow phase + self.state.ramp_start_time = timestamp + self.state.ramp_start_current = self.state.fast_ramp_target + self.state.current_limit = self.state.fast_ramp_target + new_limit = self.state.fast_ramp_target + + # Clamp to target + new_limit = min(new_limit, self.state.target_limit) + + # Clamp to absolute maximum + new_limit = min(new_limit, self.config['maximum_current']) + + # Round to nearest 0.5A + new_limit = round(new_limit * 2) / 2 + + # Check if we've reached target + if new_limit >= self.state.target_limit: + self.logger.info(f"Reached target: {new_limit}A") + self.state.ramp_start_time = None + self.state.in_fast_ramp = False + + # Start tracking stable time + if self.stable_since is None: + self.stable_since = timestamp + + # Update last stable tracking + if self.last_update_time is not None: + time_since_update = timestamp - self.last_update_time + if time_since_update >= self.config['ramp_update_interval']: + self.state.last_stable_current = max( + self.state.last_stable_current, + new_limit - 1.0 + ) + # Record stable operation for learning + self.power_model.record_stable_operation( + self.current_output_power, + new_limit, + timestamp + ) + + # Only return if changed + if abs(new_limit - self.state.current_limit) >= 0.5: + self.state.current_limit = new_limit + self.last_update_time = timestamp + return new_limit + + return None + + def handle_overload(self, timestamp: float = None, output_power: float = None) -> dict: + """ + Handle an overload event. + + Returns: + dict with: + - new_limit: Immediate limit to set (initial_current) + - recovery_target: Target for eventual recovery + - overload_count: Total overloads this session + - fast_recovery_target: Target for fast ramp phase + - is_rapid_overload: True if this was a rapid successive overload + """ + if timestamp is None: + timestamp = time() + + if output_power is None: + output_power = self.current_output_power + + # Check if this is a rapid overload (within threshold of last one) + is_rapid = False + if self.last_overload_time is not None: + time_since_last = timestamp - self.last_overload_time + if time_since_last < self.config['rapid_overload_threshold']: + is_rapid = True + self.rapid_overload_count += 1 + self.logger.warning( + f"RAPID overload detected! ({time_since_last:.0f}s since last) " + f"Rapid count: {self.rapid_overload_count}" + ) + else: + # Reset rapid count if enough time has passed + self.rapid_overload_count = 0 + + self.last_overload_time = timestamp + + # Record the overload + event = OverloadEvent( + timestamp=timestamp, + current_at_overload=self.state.current_limit, + last_stable_current=self.state.last_stable_current, + output_power=output_power + ) + self.overload_history.append(event) + + overload_count = len(self.overload_history) + + self.logger.warning( + f"OVERLOAD #{overload_count} at {event.current_at_overload}A " + f"(last stable: {event.last_stable_current}A, " + f"output power: {output_power:.0f}W)" + ) + + # Record overload in power model + self.power_model.record_overload(output_power, event.current_at_overload, timestamp) + + # Calculate new recovery target + base_margin = self.config['recovery_margin'] + extra_margin = self.config['margin_per_overload'] * (overload_count - 1) + total_margin = base_margin + extra_margin + + # Add extra margin for rapid overloads + if is_rapid: + total_margin += self.config['rapid_overload_extra_margin'] * self.rapid_overload_count + + # If we were stable at current for long enough, return to that current after recovery + # (overload likely due to step load e.g. heater; generator can handle prior level) + return_to_stable = self.config.get('return_to_stable_after_overload', False) + min_stable_duration = self.config.get('return_to_stable_min_duration', 1800) + used_return_to_stable = False + if return_to_stable and self.stable_since is not None: + stable_duration = timestamp - self.stable_since + if stable_duration >= min_stable_duration: + self.recovery_target = min( + event.current_at_overload, + self.config['target_current'] + ) + used_return_to_stable = True + self.logger.info( + f"Stable for {stable_duration/60:.1f} min at {event.current_at_overload}A: " + f"recovery target set to {self.recovery_target}A (return to prior current)" + ) + if not used_return_to_stable: + self.recovery_target = event.last_stable_current - total_margin + # Cap at (overload current - base margin) + self.recovery_target = min( + self.recovery_target, + event.current_at_overload - base_margin + ) + + # Don't go below minimum + self.recovery_target = max( + self.recovery_target, + self.config['minimum_current'] + ) + + self.logger.info( + f"Recovery target set to {self.recovery_target}A " + f"(margin: {total_margin}A)" + ) + + # Calculate fast recovery target for info + fast_margin = self.config['fast_recovery_margin'] + if is_rapid: + fast_margin += self.config['rapid_overload_extra_margin'] * self.rapid_overload_count + fast_target = max( + event.current_at_overload - fast_margin, + self.config['initial_current'] + ) + + # Reset state + self.state.current_limit = self.config['initial_current'] + self.state.ramp_start_time = None + self.state.in_fast_ramp = False + self.state.fast_ramp_target = None + self.stable_since = None + + # Track output power at overload for later comparison + self.output_power_at_last_overload = output_power + self.last_target_reevaluation = timestamp + + return { + 'new_limit': self.config['initial_current'], + 'recovery_target': self.recovery_target, + 'overload_count': overload_count, + 'fast_recovery_target': fast_target, + 'is_rapid_overload': is_rapid, + 'rapid_overload_count': self.rapid_overload_count, + } + + def check_history_clear(self, timestamp: float = None) -> bool: + """ + Check if we should clear overload history after extended stable operation. + """ + if timestamp is None: + timestamp = time() + + if not self.overload_history: + return False + + if self.stable_since is None: + return False + + stable_duration = timestamp - self.stable_since + + if stable_duration >= self.config['history_clear_time']: + self.logger.info( + f"Stable for {stable_duration/60:.1f} minutes, " + f"clearing {len(self.overload_history)} overload events" + ) + self.overload_history.clear() + self.recovery_target = self.config['target_current'] + self.rapid_overload_count = 0 + self.last_overload_time = None + return True + + return False + + def should_retry_full_power(self) -> bool: + """ + Check if we should attempt ramping to full power again. + """ + if self.overload_history: + return False + + if self.state.current_limit >= self.config['target_current']: + return False + + if self.is_ramping: + return False + + return True + + def check_output_power_increase(self, timestamp: float = None) -> Optional[dict]: + """ + Check if output power has increased significantly since last overload. + If so, re-evaluate and potentially raise the recovery target. + + Returns: + dict with new_target and should_ramp if target was raised, None otherwise + """ + if timestamp is None: + timestamp = time() + + # Only relevant if we have overload history + if not self.overload_history: + return None + + # Check minimum interval between re-evaluations + min_interval = self.learning_config.get('min_reevaluation_interval', 60) + if self.last_target_reevaluation is not None: + if timestamp - self.last_target_reevaluation < min_interval: + return None + + # Check if output power has increased significantly + threshold = self.learning_config.get('output_power_increase_threshold', 2000) + power_increase = self.current_output_power - self.output_power_at_last_overload + + if power_increase < threshold: + return None + + # Output power has increased significantly - re-evaluate target + self.logger.info( + f"Output power increased by {power_increase:.0f}W " + f"({self.output_power_at_last_overload:.0f}W -> {self.current_output_power:.0f}W)" + ) + + # Get model's prediction for current output power + if self.power_model.should_use_model(): + new_suggested = self.power_model.predict_max_current(self.current_output_power) + else: + # Without confident model, use zone-based estimate + # Higher output = can handle more input + base = self.config['target_current'] + # Add ~1A per 1000W of output power increase + bonus = power_increase / 1000.0 + new_suggested = min(base, self.recovery_target + bonus) + + # Only raise target, never lower it from this check + if new_suggested <= self.recovery_target: + self.logger.debug( + f"Model suggests {new_suggested:.1f}A, not higher than current " + f"recovery target {self.recovery_target:.1f}A" + ) + return None + + # Cap at target_current + new_target = min(new_suggested, self.config['target_current']) + + # Apply a safety margin - don't go all the way to predicted max + new_target = new_target - 1.0 + + # Round to 0.5A + new_target = round(new_target * 2) / 2 + + if new_target <= self.recovery_target: + return None + + old_target = self.recovery_target + self.recovery_target = new_target + self.last_target_reevaluation = timestamp + + # Update the reference point for future comparisons + self.output_power_at_last_overload = self.current_output_power + + self.logger.info( + f"Raised recovery target: {old_target:.1f}A -> {new_target:.1f}A " + f"(output power now {self.current_output_power:.0f}W)" + ) + + # Determine if we should start a new ramp + should_ramp = not self.is_ramping and self.state.current_limit < new_target + + return { + 'old_target': old_target, + 'new_target': new_target, + 'output_power': self.current_output_power, + 'should_ramp': should_ramp, + } + + def get_suggested_limit(self, output_power: float = None) -> float: + """ + Get the suggested current limit based on output power. + Uses the learned correlation model. + """ + if output_power is None: + output_power = self.current_output_power + + if self.power_model.should_use_model(): + suggested = self.power_model.predict_max_current(output_power) + return round(suggested * 2) / 2 + else: + return self.config['target_current'] + + def reset(self): + """Full reset (e.g., when generator stops)""" + self.state = RampState( + current_limit=self.config['initial_current'], + target_limit=self.config['target_current'], + last_stable_current=self.config['initial_current'] + ) + self.overload_history.clear() + self.recovery_target = self.config['target_current'] + self.last_update_time = None + self.stable_since = None + self.rapid_overload_count = 0 + self.last_overload_time = None + self.current_output_power = 0.0 + self.output_power_at_last_overload = 0.0 + self.last_target_reevaluation = None + self.logger.debug("Controller reset") + + def get_status(self) -> dict: + """Get current controller status""" + return { + 'current_limit': self.state.current_limit, + 'target_limit': self.state.target_limit, + 'recovery_target': self.recovery_target, + 'is_ramping': self.is_ramping, + 'is_recovery': self.state.is_recovery, + 'in_fast_ramp': self.state.in_fast_ramp, + 'fast_ramp_target': self.state.fast_ramp_target, + 'last_stable': self.state.last_stable_current, + 'overload_count': len(self.overload_history), + 'rapid_overload_count': self.rapid_overload_count, + 'stable_since': self.stable_since, + 'output_power': self.current_output_power, + 'output_power_at_overload': self.output_power_at_last_overload, + 'power_model': self.power_model.get_status(), + 'suggested_limit': self.get_suggested_limit(), + } + + +# For testing +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + + controller = RampController() + + print("=== Test: Normal Ramp ===") + controller.start_ramp(0) + for minute in range(35): + t = minute * 60 + new_limit = controller.update(t) + if new_limit or minute % 5 == 0: + status = controller.get_status() + print(f" t={minute}min: limit={status['current_limit']}A, ramping={status['is_ramping']}") + + print("\n=== Test: Overload at 45A with Fast Recovery ===") + controller.reset() + controller.set_output_power(2500) # Simulate output power + controller.start_ramp(0) + + # Ramp to 45A (about 15 minutes) + for minute in range(16): + controller.update(minute * 60) + + print(f" Before overload: limit={controller.get_status()['current_limit']}A") + + # Simulate overload at 45A + result = controller.handle_overload(16 * 60, output_power=2500) + print(f" After overload: new_limit={result['new_limit']}A, " + f"recovery_target={result['recovery_target']}A, " + f"fast_target={result['fast_recovery_target']:.1f}A") + + # After cooldown, start recovery with fast ramp + print("\n Starting recovery ramp...") + controller.start_ramp(21 * 60) # 5 min cooldown + + for second in range(0, 600, 10): # 10 minutes in 10-second increments + t = 21 * 60 + second + new_limit = controller.update(t) + if new_limit: + status = controller.get_status() + phase = "FAST" if status['in_fast_ramp'] else "SLOW" + print(f" t={21 + second/60:.1f}min: limit={status['current_limit']}A [{phase}]") + + print("\n=== Test: Rapid Successive Overloads ===") + controller.reset() + controller.set_output_power(1500) # Low output + controller.start_ramp(0) + + # Ramp to 44A + for minute in range(12): + controller.update(minute * 60) + + # First overload + result1 = controller.handle_overload(12 * 60) + print(f" 1st overload: margin=4A, fast_target={result1['fast_recovery_target']:.1f}A") + + # Quick recovery and second overload within 2 minutes + controller.start_ramp(17 * 60) + for second in range(0, 90, 10): + controller.update(17 * 60 + second) + + result2 = controller.handle_overload(18.5 * 60) # 1.5 min later + print(f" 2nd RAPID overload: margin=6A, fast_target={result2['fast_recovery_target']:.1f}A, " + f"is_rapid={result2['is_rapid_overload']}") + + # Third overload also rapid + controller.start_ramp(24 * 60) + for second in range(0, 60, 10): + controller.update(24 * 60 + second) + + result3 = controller.handle_overload(25 * 60) # 1 min later + print(f" 3rd RAPID overload: margin=8A, fast_target={result3['fast_recovery_target']:.1f}A, " + f"rapid_count={result3['rapid_overload_count']}") + + print("\n=== Test: Power Model Suggestions ===") + model = controller.power_model + print(f" Model status: {model.get_status()}") + print(f" Suggested limit at 1500W output: {controller.get_suggested_limit(1500):.1f}A") + print(f" Suggested limit at 4000W output: {controller.get_suggested_limit(4000):.1f}A") + print(f" Suggested limit at 6000W output: {controller.get_suggested_limit(6000):.1f}A") diff --git a/dbus-generator-ramp/service-webui/._log b/dbus-generator-ramp/service-webui/._log new file mode 100755 index 0000000..57fc954 Binary files /dev/null and b/dbus-generator-ramp/service-webui/._log differ diff --git a/dbus-generator-ramp/service-webui/._run b/dbus-generator-ramp/service-webui/._run new file mode 100755 index 0000000..57fc954 Binary files /dev/null and b/dbus-generator-ramp/service-webui/._run differ diff --git a/dbus-generator-ramp/service-webui/log/._run b/dbus-generator-ramp/service-webui/log/._run new file mode 100755 index 0000000..57fc954 Binary files /dev/null and b/dbus-generator-ramp/service-webui/log/._run differ diff --git a/dbus-generator-ramp/service-webui/log/run b/dbus-generator-ramp/service-webui/log/run new file mode 100755 index 0000000..61aa2e8 --- /dev/null +++ b/dbus-generator-ramp/service-webui/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec multilog t s99999 n8 /var/log/dbus-generator-ramp-webui diff --git a/dbus-generator-ramp/service-webui/run b/dbus-generator-ramp/service-webui/run new file mode 100755 index 0000000..b935273 --- /dev/null +++ b/dbus-generator-ramp/service-webui/run @@ -0,0 +1,9 @@ +#!/bin/sh +# +# daemontools run script for Generator Ramp Controller Web UI +# + +exec 2>&1 +cd /data/dbus-generator-ramp +export PYTHONPATH="/opt/victronenergy/velib_python:$PYTHONPATH" +exec python3 /data/dbus-generator-ramp/web_ui.py diff --git a/dbus-generator-ramp/service/._log b/dbus-generator-ramp/service/._log new file mode 100755 index 0000000..90e3722 Binary files /dev/null and b/dbus-generator-ramp/service/._log differ diff --git a/dbus-generator-ramp/service/._run b/dbus-generator-ramp/service/._run new file mode 100755 index 0000000..90e3722 Binary files /dev/null and b/dbus-generator-ramp/service/._run differ diff --git a/dbus-generator-ramp/service/log/._run b/dbus-generator-ramp/service/log/._run new file mode 100755 index 0000000..57fc954 Binary files /dev/null and b/dbus-generator-ramp/service/log/._run differ diff --git a/dbus-generator-ramp/service/log/run b/dbus-generator-ramp/service/log/run new file mode 100755 index 0000000..bc03d17 --- /dev/null +++ b/dbus-generator-ramp/service/log/run @@ -0,0 +1,8 @@ +#!/bin/sh +# +# daemontools log run script +# +# Logs service output using multilog +# + +exec multilog t s99999 n8 /var/log/dbus-generator-ramp diff --git a/dbus-generator-ramp/service/run b/dbus-generator-ramp/service/run new file mode 100755 index 0000000..4773b0d --- /dev/null +++ b/dbus-generator-ramp/service/run @@ -0,0 +1,18 @@ +#!/bin/sh +# +# daemontools run script for Generator Current Ramp Controller +# +# This script is called by daemontools to start the service. +# It will be automatically restarted if it exits. +# + +exec 2>&1 + +# Change to the script directory +cd /data/dbus-generator-ramp + +# Add velib_python to path (use the symlink created by install.sh) +export PYTHONPATH="/data/dbus-generator-ramp/ext/velib_python:$PYTHONPATH" + +# Execute the main script +exec python3 /data/dbus-generator-ramp/dbus-generator-ramp.py diff --git a/dbus-generator-ramp/uninstall.sh b/dbus-generator-ramp/uninstall.sh new file mode 100755 index 0000000..e0bb1d8 --- /dev/null +++ b/dbus-generator-ramp/uninstall.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# +# Uninstallation script for Generator Current Ramp Controller +# +# Usage: +# chmod +x uninstall.sh +# ./uninstall.sh +# + +set -e + +INSTALL_DIR="/data/dbus-generator-ramp" + +# Determine the correct service directory (same logic as install.sh) +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +elif [ -L "/service" ]; then + SERVICE_DIR=$(readlink -f /service) +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "==================================================" +echo "Generator Current Ramp Controller - Uninstallation" +echo "==================================================" +echo "Using service directory: $SERVICE_DIR" + +echo "1. Stopping service and supervise..." +# Try svc if service directory exists +if [ -d "$SERVICE_DIR/dbus-generator-ramp" ] || [ -L "$SERVICE_DIR/dbus-generator-ramp" ]; then + # -d = down (stop service), -x = exit (stop supervise) + svc -dx "$SERVICE_DIR/dbus-generator-ramp" 2>/dev/null || true + sleep 2 + echo " Sent stop signal via svc" +fi + +# Kill any remaining processes (in case symlink was already removed or svc failed) +# Kill the python script first +pkill -f "python.*dbus-generator-ramp.py" 2>/dev/null && echo " Killed python process" || true +# Kill the supervise process +pkill -f "supervise dbus-generator-ramp$" 2>/dev/null && echo " Killed supervise process" || true +# Kill multilog for this service (use simple pattern - busybox pkill has limited regex) +pkill -f "/var/log/dbus-generator-ramp" 2>/dev/null && echo " Killed multilog process" || true +sleep 1 + +echo "2. Removing service symlink..." +if [ -L "$SERVICE_DIR/dbus-generator-ramp" ]; then + rm "$SERVICE_DIR/dbus-generator-ramp" + echo " Removed service link" +elif [ -d "$SERVICE_DIR/dbus-generator-ramp" ]; then + rm -rf "$SERVICE_DIR/dbus-generator-ramp" + echo " Removed service directory" +else + echo " Service link not found" +fi + +echo "3. Stopping and removing web UI service (if installed)..." +if [ -d "$SERVICE_DIR/dbus-generator-ramp-webui" ] || [ -L "$SERVICE_DIR/dbus-generator-ramp-webui" ]; then + svc -dx "$SERVICE_DIR/dbus-generator-ramp-webui" 2>/dev/null || true + sleep 1 + if [ -L "$SERVICE_DIR/dbus-generator-ramp-webui" ]; then + rm "$SERVICE_DIR/dbus-generator-ramp-webui" + elif [ -d "$SERVICE_DIR/dbus-generator-ramp-webui" ]; then + rm -rf "$SERVICE_DIR/dbus-generator-ramp-webui" + fi + echo " Web UI service removed" +fi +# Kill any remaining webui processes +pkill -f "python.*web_ui.py" 2>/dev/null || true +pkill -f "supervise dbus-generator-ramp-webui" 2>/dev/null || true + +echo "4. Note: Not removing files from $INSTALL_DIR" +echo " To fully remove, run: rm -rf $INSTALL_DIR" + +echo "5. Note: rc.local entry not removed" +echo " Edit /data/rc.local manually if desired" + +echo "" +echo "==================================================" +echo "Uninstallation complete!" +echo "==================================================" +echo "" +echo "The service has been stopped and disabled." +echo "Files remain in $INSTALL_DIR" +echo "" diff --git a/dbus-generator-ramp/web_ui.py b/dbus-generator-ramp/web_ui.py new file mode 100755 index 0000000..51b80f7 --- /dev/null +++ b/dbus-generator-ramp/web_ui.py @@ -0,0 +1,645 @@ +#!/usr/bin/env python3 +""" +Simple Web UI for Generator Ramp Controller + +Provides a browser-based interface for: +- Viewing current status +- Adjusting settings +- Monitoring power and ramp progress + +Access at: http://:8088 + +This runs as a separate service alongside the main controller. +It communicates via D-Bus to read/write settings. +""" + +import sys +import os +import json +import logging +from http.server import HTTPServer, BaseHTTPRequestHandler +from urllib.parse import parse_qs, urlparse +import threading + +# Add velib_python to path +sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'ext', 'velib_python')) +sys.path.insert(1, '/opt/victronenergy/velib_python') + +try: + import dbus +except ImportError: + print("ERROR: dbus-python not available") + sys.exit(1) + +# Configuration +WEB_PORT = 8088 +SERVICE_NAME = 'com.victronenergy.generatorramp' + +logger = logging.getLogger('WebUI') + + +class DBusClient: + """Client to read/write from our D-Bus service""" + + def __init__(self): + self.bus = dbus.SystemBus() + + def get_value(self, path): + """Get a value from our service""" + try: + obj = self.bus.get_object(SERVICE_NAME, path, introspect=False) + return obj.GetValue(dbus_interface='com.victronenergy.BusItem') + except Exception as e: + logger.warning(f"Failed to get {path}: {e}") + return None + + def set_value(self, path, value): + """Set a value on our service""" + try: + obj = self.bus.get_object(SERVICE_NAME, path, introspect=False) + obj.SetValue(value, dbus_interface='com.victronenergy.BusItem') + return True + except Exception as e: + logger.error(f"Failed to set {path}: {e}") + return False + + def get_all_status(self): + """Get all status values""" + paths = [ + '/State', + '/CurrentLimit', + '/TargetLimit', + '/RecoveryTarget', + '/OverloadCount', + '/LastStableCurrent', + '/Power/L1', + '/Power/L2', + '/Power/Total', + '/Ramp/Progress', + '/Ramp/TimeRemaining', + '/Detection/Reversals', + '/Detection/StdDev', + '/Detection/IsOverload', + '/Generator/State', + '/AcInput/Connected', + '/Settings/InitialCurrent', + '/Settings/TargetCurrent', + '/Settings/RampDuration', + '/Settings/CooldownDuration', + '/Settings/Enabled', + ] + + result = {} + for path in paths: + value = self.get_value(path) + # Convert path to key (remove leading /) + key = path[1:].replace('/', '_') + result[key] = value + + return result + + +# HTML Template +HTML_TEMPLATE = ''' + + + + + Generator Ramp Controller + + + +
+

Generator Ramp Controller

+ + +
+

Current State

+
+ Idle +
+ Controller: + + Enabled +
+
+ +
+
+
Current Limit
+
--
+
+
+
Target
+
--
+
+
+
Overload Count
+
0
+
+
+
Generator
+
--
+
+
+ + +
+
+ Ramp Progress + 0% +
+
+
+
+
+ Time remaining: -- +
+
+
+ + +
+

Power Monitoring

+
+
+
0
+
L1 (W)
+
+
+
0
+
L2 (W)
+
+
+
0
+
Total (W)
+
+
+ +
+
+
+
Overload Detection: Normal
+
+ Reversals: 0 | + Std Dev: 0W +
+
+
+
+ + +
+

Settings

+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ +
+ Auto-refreshes every 2 seconds | Last update: -- +
+
+ + + + +''' + + +class WebHandler(BaseHTTPRequestHandler): + """HTTP request handler""" + + dbus_client = None + + def log_message(self, format, *args): + logger.debug(f"{self.address_string()} - {format % args}") + + def send_json(self, data, status=200): + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.send_header('Access-Control-Allow-Origin', '*') + self.end_headers() + self.wfile.write(json.dumps(data).encode()) + + def send_html(self, html): + self.send_response(200) + self.send_header('Content-Type', 'text/html') + self.end_headers() + self.wfile.write(html.encode()) + + def do_GET(self): + parsed = urlparse(self.path) + + if parsed.path == '/': + self.send_html(HTML_TEMPLATE) + + elif parsed.path == '/api/status': + if self.dbus_client is None: + self.dbus_client = DBusClient() + status = self.dbus_client.get_all_status() + self.send_json(status) + + else: + self.send_response(404) + self.end_headers() + + def do_POST(self): + parsed = urlparse(self.path) + + if parsed.path == '/api/settings': + content_length = int(self.headers.get('Content-Length', 0)) + body = self.rfile.read(content_length) + + try: + settings = json.loads(body) + if self.dbus_client is None: + self.dbus_client = DBusClient() + + for path, value in settings.items(): + self.dbus_client.set_value(path, value) + + self.send_json({'status': 'ok'}) + except Exception as e: + logger.error(f"Failed to save settings: {e}") + self.send_json({'error': str(e)}, 500) + else: + self.send_response(404) + self.end_headers() + + def do_OPTIONS(self): + self.send_response(200) + self.send_header('Access-Control-Allow-Origin', '*') + self.send_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS') + self.send_header('Access-Control-Allow-Headers', 'Content-Type') + self.end_headers() + + +def run_server(): + """Run the web server""" + logging.basicConfig( + level=logging.INFO, + format='%(levelname)s %(name)s: %(message)s' + ) + + server = HTTPServer(('0.0.0.0', WEB_PORT), WebHandler) + logger.info(f"Web UI running at http://0.0.0.0:{WEB_PORT}") + + try: + server.serve_forever() + except KeyboardInterrupt: + logger.info("Shutting down web server") + server.shutdown() + + +if __name__ == '__main__': + run_server() diff --git a/dbus-lightning/.gitignore b/dbus-lightning/.gitignore new file mode 100644 index 0000000..5923ff5 --- /dev/null +++ b/dbus-lightning/.gitignore @@ -0,0 +1,24 @@ +# Build artifacts +*.tar.gz +*.sha256 + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg +*.egg-info/ +dist/ +build/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Venus OS runtime (created during installation) +ext/ diff --git a/dbus-lightning/README.md b/dbus-lightning/README.md new file mode 100644 index 0000000..8359703 --- /dev/null +++ b/dbus-lightning/README.md @@ -0,0 +1,45 @@ +# dbus-lightning + +Venus OS D-Bus service that monitors real-time lightning strikes from the [Blitzortung](https://www.blitzortung.org/) community lightning detection network. + +## Features + +- Connects to Blitzortung WebSocket servers for real-time strike data +- Filters strikes by configurable radius from current GPS position +- Analyzes 15-minute windows for storm intensity tracking +- Detects approaching storms via linear regression on distance trends +- Estimates storm arrival time (ETA) +- Publishes all data to D-Bus for display in the venus-html5-app dashboard + +## Architecture + +| Module | Purpose | +|--------|---------| +| `lightning.py` | Main controller, D-Bus service, GPS reader | +| `config.py` | Service configuration (servers, radii, thresholds) | +| `blitzortung_client.py` | WebSocket client with LZW decoding and auto-reconnect | +| `strike_buffer.py` | Thread-safe strike buffer with haversine distance/bearing | +| `analysis_engine.py` | Windowed analysis, approach detection, ETA calculation | + +## Installation + +```bash +./build-package.sh +scp dbus-lightning-*.tar.gz root@:/data/ +ssh root@ +cd /data && tar -xzf dbus-lightning-*.tar.gz +bash /data/dbus-lightning/install.sh +``` + +## D-Bus Service + +- **Service name:** `com.victronenergy.lightning` +- **Key paths:** `/StrikeCount15m`, `/NearestStrikeMiles`, `/StormApproaching`, `/StormEtaMinutes`, `/Settings/Enabled` + +## Service Management + +```bash +svstat /service/dbus-lightning +svc -t /service/dbus-lightning # restart +tail -f /var/log/dbus-lightning/current | tai64nlocal +``` diff --git a/dbus-lightning/analysis_engine.py b/dbus-lightning/analysis_engine.py new file mode 100644 index 0000000..290979e --- /dev/null +++ b/dbus-lightning/analysis_engine.py @@ -0,0 +1,223 @@ +""" +Lightning analysis engine: windowed aggregation, approach detection, ETA. +""" + +import math +import time + +from strike_buffer import haversine_miles, bearing_degrees + + +def _linear_regression(xs, ys): + """Simple linear regression returning (slope, intercept, r_squared).""" + n = len(xs) + if n < 2: + return None, None, None + + sum_x = sum(xs) + sum_y = sum(ys) + sum_xy = sum(x * y for x, y in zip(xs, ys)) + sum_x2 = sum(x * x for x in xs) + sum_y2 = sum(y * y for y in ys) + + denom = n * sum_x2 - sum_x * sum_x + if abs(denom) < 1e-12: + return None, None, None + + slope = (n * sum_xy - sum_x * sum_y) / denom + intercept = (sum_y - slope * sum_x) / n + + ss_tot = sum_y2 - (sum_y * sum_y) / n + if abs(ss_tot) < 1e-12: + return slope, intercept, 1.0 + + ss_res = sum((y - (slope * x + intercept)) ** 2 for x, y in zip(xs, ys)) + r_squared = max(0.0, 1.0 - ss_res / ss_tot) + + return slope, intercept, r_squared + + +def _bearing_stddev(bearings): + """Circular standard deviation of bearings in degrees.""" + if len(bearings) < 2: + return 0.0 + to_rad = math.pi / 180 + sin_sum = sum(math.sin(b * to_rad) for b in bearings) + cos_sum = sum(math.cos(b * to_rad) for b in bearings) + n = len(bearings) + r = math.sqrt((sin_sum / n) ** 2 + (cos_sum / n) ** 2) + r = min(r, 1.0) + if r >= 1.0: + return 0.0 + return math.degrees(math.sqrt(-2.0 * math.log(r))) + + +def _bearing_to_cardinal(bearing): + """Convert bearing to cardinal/intercardinal label.""" + dirs = ['N', 'NE', 'E', 'SE', 'S', 'SW', 'W', 'NW'] + idx = int(((bearing + 22.5) % 360) / 45) + return dirs[idx] + + +class AnalysisEngine: + """Processes strike buffer into a summary for frontend display.""" + + def __init__(self, config): + self.window_size = config.get('window_size', 900) + self.num_windows = config.get('num_windows', 4) + self.min_strikes_active = config.get('min_strikes_active', 3) + self.min_windows_approach = config.get('min_windows_approach', 3) + self.r2_approaching = config.get('r2_approaching', 0.7) + self.r2_downgrade = config.get('r2_downgrade', 0.5) + self.max_bearing_stddev = config.get('max_bearing_stddev', 30.0) + self.max_eta_minutes = config.get('max_eta_minutes', 240) + + def analyze(self, strikes, vessel_lat, vessel_lon): + """Produce summary dict from current strike list and vessel position.""" + now = time.time() + now_ms = now * 1000 + + recent_cutoff = now_ms - self.window_size * 1000 + recent = [s for s in strikes if s.timestamp >= recent_cutoff] + + active = len(recent) >= self.min_strikes_active + + if not active: + return { + 'active': False, + 'strike_count_15m': len(recent), + 'nearest_distance': None, + 'centroid_bearing': None, + 'centroid_distance': None, + 'cardinal': None, + 'approaching': False, + 'approach_speed': None, + 'eta_minutes': None, + 'confidence': None, + } + + nearest_distance = min(s.distance for s in recent) if recent else None + + windows = self._build_windows(strikes, now_ms) + + current_window = windows[-1] if windows else None + centroid_bearing = None + centroid_distance = None + cardinal = None + + if current_window and current_window['count'] > 0: + centroid_bearing = current_window['centroid_bearing'] + centroid_distance = current_window['centroid_distance'] + cardinal = _bearing_to_cardinal(centroid_bearing) + + approaching, approach_speed, eta_minutes, confidence = \ + self._detect_approach(windows, vessel_lat, vessel_lon) + + if centroid_distance is not None: + centroid_distance = round(centroid_distance, 1) + if nearest_distance is not None: + nearest_distance = round(nearest_distance, 1) + if centroid_bearing is not None: + centroid_bearing = round(centroid_bearing, 0) + if approach_speed is not None: + approach_speed = round(approach_speed, 0) + if eta_minutes is not None: + eta_minutes = round(eta_minutes, 0) + if confidence is not None: + confidence = round(confidence, 2) + + return { + 'active': True, + 'strike_count_15m': len(recent), + 'nearest_distance': nearest_distance, + 'centroid_bearing': centroid_bearing, + 'centroid_distance': centroid_distance, + 'cardinal': cardinal, + 'approaching': approaching, + 'approach_speed': approach_speed, + 'eta_minutes': eta_minutes, + 'confidence': confidence, + } + + def _build_windows(self, strikes, now_ms): + """Divide strikes into 15-minute windows, most recent last.""" + windows = [] + for i in range(self.num_windows): + window_end = now_ms - i * self.window_size * 1000 + window_start = window_end - self.window_size * 1000 + window_mid = (window_start + window_end) / 2 + + window_strikes = [s for s in strikes + if window_start <= s.timestamp < window_end] + + if not window_strikes: + windows.append({ + 'mid_time': window_mid, + 'count': 0, + 'centroid_lat': None, + 'centroid_lon': None, + 'centroid_distance': None, + 'centroid_bearing': None, + }) + continue + + avg_lat = sum(s.lat for s in window_strikes) / len(window_strikes) + avg_lon = sum(s.lon for s in window_strikes) / len(window_strikes) + avg_dist = sum(s.distance for s in window_strikes) / len(window_strikes) + + to_rad = math.pi / 180 + sin_sum = sum(math.sin(s.bearing * to_rad) for s in window_strikes) + cos_sum = sum(math.cos(s.bearing * to_rad) for s in window_strikes) + avg_bearing = (math.degrees(math.atan2(sin_sum, cos_sum)) + 360) % 360 + + windows.append({ + 'mid_time': window_mid, + 'count': len(window_strikes), + 'centroid_lat': avg_lat, + 'centroid_lon': avg_lon, + 'centroid_distance': avg_dist, + 'centroid_bearing': avg_bearing, + }) + + windows.reverse() + return windows + + def _detect_approach(self, windows, vessel_lat, vessel_lon): + """Detect if storm is approaching. Returns (approaching, speed_mph, eta_min, r2).""" + populated = [w for w in windows if w['count'] > 0] + if len(populated) < self.min_windows_approach: + return False, None, None, None + + times = [w['mid_time'] / 60000 for w in populated] + distances = [w['centroid_distance'] for w in populated] + bearings = [w['centroid_bearing'] for w in populated] + + bearing_sd = _bearing_stddev(bearings) + if bearing_sd > self.max_bearing_stddev: + return False, None, None, None + + slope, intercept, r2 = _linear_regression(times, distances) + if slope is None or r2 is None: + return False, None, None, None + + if slope >= 0: + return False, None, None, None + + if r2 < self.r2_downgrade: + return False, None, None, None + + approaching = r2 >= self.r2_approaching + if not approaching: + return False, None, None, r2 + + speed_mph = abs(slope) * 60 + current_distance = populated[-1]['centroid_distance'] + + if speed_mph < 0.1: + return True, 0, None, r2 + + eta = current_distance / speed_mph * 60 + if eta > self.max_eta_minutes: + eta = self.max_eta_minutes + + return True, speed_mph, eta, r2 diff --git a/dbus-lightning/blitzortung_client.py b/dbus-lightning/blitzortung_client.py new file mode 100644 index 0000000..5262176 --- /dev/null +++ b/dbus-lightning/blitzortung_client.py @@ -0,0 +1,378 @@ +""" +Blitzortung.org WebSocket client with LZW decoding and automatic reconnection. + +Uses only Python stdlib (socket, ssl, hashlib, struct) -- no external dependencies. +Implements the WebSocket protocol (RFC 6455) at the minimum level needed for +text-frame communication with the Blitzortung push service. +""" + +import hashlib +import json +import logging +import os +import socket +import ssl +import struct +import threading +import time +from base64 import b64encode + +logger = logging.getLogger('BlitzortungClient') + +STATUS_CONNECTED = 'connected' +STATUS_DISCONNECTED = 'disconnected' +STATUS_RECONNECTING = 'reconnecting' + +_WS_MAGIC = b'258EAFA5-E914-47DA-95CA-5AB4B141E427' + +OPCODE_TEXT = 0x1 +OPCODE_CLOSE = 0x8 +OPCODE_PING = 0x9 +OPCODE_PONG = 0xA + + +def decode_lzw(b): + """Decode LZW-obfuscated Blitzortung message into a dict.""" + e = {} + d = list(b) + c = d[0] + f = c + g = [c] + h = 256 + o = h + for i in range(1, len(d)): + a = ord(d[i]) + a = d[i] if h > a else e[a] if e.get(a) else f + c + g.append(a) + c = a[0] + e[o] = f + c + o += 1 + f = a + return json.loads(''.join(g)) + + +def _parse_wss_url(url): + """Parse 'wss://host[:port][/path]' into (host, port, path).""" + if url.startswith('wss://'): + rest = url[6:] + elif url.startswith('ws://'): + rest = url[5:] + else: + raise ValueError(f"Unsupported URL scheme: {url}") + + is_tls = url.startswith('wss://') + default_port = 443 if is_tls else 80 + + slash = rest.find('/') + if slash >= 0: + host_part = rest[:slash] + path = rest[slash:] + else: + host_part = rest + path = '/' + + if ':' in host_part: + host, port_str = host_part.rsplit(':', 1) + port = int(port_str) + else: + host = host_part + port = default_port + + return host, port, path, is_tls + + +class _WebSocketConn: + """Minimal RFC 6455 WebSocket client using stdlib only.""" + + def __init__(self, url, timeout=30): + self._sock = None + self._closed = False + host, port, path, is_tls = _parse_wss_url(url) + self._connect(host, port, path, is_tls, timeout) + + def _connect(self, host, port, path, is_tls, timeout): + raw = socket.create_connection((host, port), timeout=timeout) + raw.settimeout(timeout) + + if is_tls: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + self._sock = ctx.wrap_socket(raw, server_hostname=host) + else: + self._sock = raw + + key_bytes = b64encode(os.urandom(16)) + key = key_bytes.decode('ascii') + request = ( + f"GET {path} HTTP/1.1\r\n" + f"Host: {host}\r\n" + f"Upgrade: websocket\r\n" + f"Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {key}\r\n" + f"Sec-WebSocket-Version: 13\r\n" + f"Origin: https://{host}\r\n" + f"\r\n" + ) + self._sock.sendall(request.encode('ascii')) + + response = b'' + while b'\r\n\r\n' not in response: + chunk = self._sock.recv(4096) + if not chunk: + raise ConnectionError("Connection closed during handshake") + response += chunk + + header_block = response.split(b'\r\n\r\n')[0].decode('ascii', errors='replace') + status_line = header_block.split('\r\n')[0] + if '101' not in status_line: + raise ConnectionError(f"WebSocket handshake failed: {status_line}") + + expected_accept = b64encode( + hashlib.sha1(key_bytes + _WS_MAGIC).digest() + ).decode('ascii') + + def send_text(self, text): + """Send a masked text frame.""" + payload = text.encode('utf-8') + frame = bytearray() + frame.append(0x80 | OPCODE_TEXT) + + length = len(payload) + if length < 126: + frame.append(0x80 | length) + elif length < 65536: + frame.append(0x80 | 126) + frame.extend(struct.pack('!H', length)) + else: + frame.append(0x80 | 127) + frame.extend(struct.pack('!Q', length)) + + mask = os.urandom(4) + frame.extend(mask) + frame.extend(bytes(b ^ mask[i % 4] for i, b in enumerate(payload))) + + self._sock.sendall(frame) + + def _recv_exact(self, n): + """Read exactly n bytes from the socket.""" + data = b'' + while len(data) < n: + chunk = self._sock.recv(n - len(data)) + if not chunk: + raise ConnectionError("Connection closed") + data += chunk + return data + + def recv(self): + """Receive next text message, handling control frames internally. + + Returns the text payload as a string, or None on close. + Raises socket.timeout on timeout. + """ + buf = bytearray() + + while True: + b0, b1 = self._recv_exact(2) + fin = b0 & 0x80 + opcode = b0 & 0x0F + masked = b1 & 0x80 + length = b1 & 0x7F + + if length == 126: + length = struct.unpack('!H', self._recv_exact(2))[0] + elif length == 127: + length = struct.unpack('!Q', self._recv_exact(8))[0] + + if masked: + mask = self._recv_exact(4) + raw = self._recv_exact(length) + payload = bytes(b ^ mask[i % 4] for i, b in enumerate(raw)) + else: + payload = self._recv_exact(length) + + if opcode == OPCODE_CLOSE: + self._send_close() + return None + elif opcode == OPCODE_PING: + self._send_pong(payload) + continue + elif opcode == OPCODE_PONG: + continue + elif opcode == OPCODE_TEXT or opcode == 0x0: + buf.extend(payload) + if fin: + return buf.decode('utf-8', errors='replace') + + def _send_close(self): + """Send a close frame.""" + try: + frame = bytes([0x80 | OPCODE_CLOSE, 0x80, 0, 0, 0, 0]) + self._sock.sendall(frame) + except Exception: + pass + + def _send_pong(self, data): + """Send a pong frame echoing the ping payload.""" + frame = bytearray() + frame.append(0x80 | OPCODE_PONG) + length = len(data) + if length < 126: + frame.append(0x80 | length) + else: + frame.append(0x80 | 126) + frame.extend(struct.pack('!H', length)) + mask = os.urandom(4) + frame.extend(mask) + frame.extend(bytes(b ^ mask[i % 4] for i, b in enumerate(data))) + try: + self._sock.sendall(frame) + except Exception: + pass + + def close(self): + if self._closed: + return + self._closed = True + try: + self._send_close() + except Exception: + pass + try: + self._sock.shutdown(socket.SHUT_RDWR) + except Exception: + pass + try: + self._sock.close() + except Exception: + pass + + +class BlitzortungClient: + """Persistent WebSocket connection to Blitzortung with auto-reconnect.""" + + def __init__(self, servers, init_msg, on_strike, base_delay=1.0, max_delay=60.0): + self.servers = servers + self.init_msg = init_msg + self.on_strike = on_strike + self.base_delay = base_delay + self.max_delay = max_delay + + self.status = STATUS_DISCONNECTED + self._server_index = 0 + self._delay = base_delay + self._running = False + self._thread = None + self._ws = None + self._lock = threading.Lock() + + def start(self): + if self._running: + return + self._running = True + self._thread = threading.Thread(target=self._run_loop, daemon=True) + self._thread.start() + logger.info("Client started") + + def stop(self): + self._running = False + with self._lock: + if self._ws: + try: + self._ws.close() + except Exception: + pass + if self._thread: + self._thread.join(timeout=5) + self.status = STATUS_DISCONNECTED + logger.info("Client stopped") + + def _run_loop(self): + while self._running: + url = self.servers[self._server_index] + try: + self._connect(url) + except Exception as e: + logger.warning(f"Connection to {url} failed: {e}") + + if not self._running: + break + + self.status = STATUS_RECONNECTING + logger.info(f"Reconnecting in {self._delay:.0f}s...") + time.sleep(self._delay) + + self._delay = min(self._delay * 2, self.max_delay) + self._server_index = (self._server_index + 1) % len(self.servers) + + def _connect(self, url): + logger.info(f"Connecting to {url}") + + try: + ws = _WebSocketConn(url, timeout=30) + except Exception as e: + logger.warning(f"WebSocket connect error: {e}") + raise + + with self._lock: + self._ws = ws + + self.status = STATUS_CONNECTED + self._delay = self.base_delay + logger.info(f"Connected to {url}") + + try: + ws.send_text(self.init_msg) + logger.info(f"Sent init message: {self.init_msg}") + except Exception as e: + logger.error(f"Failed to send init: {e}") + ws.close() + raise + + while self._running: + try: + raw = ws.recv() + if raw is None: + logger.warning("WebSocket connection closed by server") + break + self._handle_message(raw) + except socket.timeout: + continue + except (ConnectionError, OSError) as e: + logger.warning(f"WebSocket recv error: {e}") + break + + self.status = STATUS_DISCONNECTED + with self._lock: + self._ws = None + ws.close() + + def _handle_message(self, raw): + try: + data = decode_lzw(raw) + + if not isinstance(data, dict): + return + + lat = data.get('lat') + lon = data.get('lon') + strike_time = data.get('time') + + if lat is None or lon is None: + return + + if strike_time is not None: + if strike_time > 1e15: + ts_ms = int(strike_time / 1e6) + elif strike_time > 1e12: + ts_ms = int(strike_time) + else: + ts_ms = int(strike_time * 1000) + else: + ts_ms = int(time.time() * 1000) + + self.on_strike(float(lat), float(lon), ts_ms) + except json.JSONDecodeError: + pass + except Exception as e: + logger.debug(f"Message decode error: {e}") diff --git a/dbus-lightning/build-package.sh b/dbus-lightning/build-package.sh new file mode 100755 index 0000000..6ab252f --- /dev/null +++ b/dbus-lightning/build-package.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# +# Build script for Lightning Monitor Venus OS package +# +# Usage: +# ./build-package.sh +# ./build-package.sh --version 1.0.0 +# +# Installation on Venus OS: +# scp dbus-lightning-*.tar.gz root@:/data/ +# ssh root@ +# cd /data && tar -xzf dbus-lightning-*.tar.gz +# bash /data/dbus-lightning/install.sh +# + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +VERSION="1.0.0" +OUTPUT_DIR="$SCRIPT_DIR" +PACKAGE_NAME="dbus-lightning" + +while [[ $# -gt 0 ]]; do + case $1 in + --version|-v) + VERSION="$2" + shift 2 + ;; + --output|-o) + OUTPUT_DIR="$2" + shift 2 + ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " -v, --version VERSION Set package version (default: 1.0.0)" + echo " -o, --output PATH Output directory (default: script directory)" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +BUILD_DATE=$(date -u +"%Y-%m-%d %H:%M:%S UTC") +BUILD_TIMESTAMP=$(date +%Y%m%d%H%M%S) + +BUILD_DIR=$(mktemp -d) +PACKAGE_DIR="$BUILD_DIR/$PACKAGE_NAME" + +echo "==================================================" +echo "Building $PACKAGE_NAME package" +echo "==================================================" +echo "Version: $VERSION" +echo "Build date: $BUILD_DATE" +echo "Source: $SCRIPT_DIR" +echo "Output: $OUTPUT_DIR" +echo "" + +echo "1. Creating package structure..." +mkdir -p "$PACKAGE_DIR" +mkdir -p "$PACKAGE_DIR/service/log" + +[ "$(uname)" = "Darwin" ] && export COPYFILE_DISABLE=1 + +echo "2. Copying application files..." +cp "$SCRIPT_DIR/lightning.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/config.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/blitzortung_client.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/strike_buffer.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/analysis_engine.py" "$PACKAGE_DIR/" + +echo "3. Copying service files..." +cp "$SCRIPT_DIR/service/run" "$PACKAGE_DIR/service/" +cp "$SCRIPT_DIR/service/log/run" "$PACKAGE_DIR/service/log/" + +echo "4. Copying installation scripts..." +cp "$SCRIPT_DIR/install.sh" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/uninstall.sh" "$PACKAGE_DIR/" + +echo "5. Setting permissions..." +chmod +x "$PACKAGE_DIR/lightning.py" +chmod +x "$PACKAGE_DIR/install.sh" +chmod +x "$PACKAGE_DIR/uninstall.sh" +chmod +x "$PACKAGE_DIR/service/run" +chmod +x "$PACKAGE_DIR/service/log/run" + +mkdir -p "$OUTPUT_DIR" + +TARBALL_NAME="$PACKAGE_NAME-$VERSION.tar.gz" +OUTPUT_DIR_ABS="$(cd "$OUTPUT_DIR" && pwd)" +TARBALL_PATH="$OUTPUT_DIR_ABS/$TARBALL_NAME" + +echo "6. Creating package archive..." +cd "$BUILD_DIR" +if [ "$(uname)" = "Darwin" ]; then + if command -v xattr >/dev/null 2>&1; then + xattr -cr "$PACKAGE_NAME" + fi +fi +tar --format=ustar -czf "$TARBALL_PATH" "$PACKAGE_NAME" + +if command -v sha256sum >/dev/null 2>&1; then + CHECKSUM=$(sha256sum "$TARBALL_PATH" | cut -d' ' -f1) +else + CHECKSUM=$(shasum -a 256 "$TARBALL_PATH" | cut -d' ' -f1) +fi +echo "$CHECKSUM $TARBALL_NAME" > "$OUTPUT_DIR_ABS/$TARBALL_NAME.sha256" + +echo "7. Cleaning up..." +rm -rf "$BUILD_DIR" + +FILE_SIZE=$(du -h "$TARBALL_PATH" | cut -f1) + +echo "" +echo "==================================================" +echo "Build complete!" +echo "==================================================" +echo "" +echo "Package: $TARBALL_PATH" +echo "Size: $FILE_SIZE" +echo "SHA256: $CHECKSUM" +echo "" +echo "Installation on Venus OS:" +echo " scp $TARBALL_PATH root@:/data/" +echo " ssh root@" +echo " cd /data" +echo " tar -xzf $TARBALL_NAME" +echo " bash /data/$PACKAGE_NAME/install.sh" +echo "" diff --git a/dbus-lightning/config.py b/dbus-lightning/config.py new file mode 100644 index 0000000..de8a1b8 --- /dev/null +++ b/dbus-lightning/config.py @@ -0,0 +1,51 @@ +""" +Configuration for Lightning Monitor Venus OS service. +""" + +SERVICE_NAME = 'com.victronenergy.lightning' + +# Blitzortung WebSocket endpoints (cycle through on failure) +BLITZORTUNG_SERVERS = [ + 'wss://ws1.blitzortung.org', + 'wss://ws5.blitzortung.org', + 'wss://ws6.blitzortung.org', + 'wss://ws7.blitzortung.org', +] + +BLITZORTUNG_INIT_MSG = '{"a": 111}' + +STRIKE_RADIUS_MILES = 500.0 # TODO: restore to 75.0 after testing + +STRIKE_MAX_AGE_SECONDS = 7200 # 2 hours + +MIN_STRIKES_ACTIVE = 1 # TODO: restore to 3 after testing + +ANALYSIS_INTERVAL_SECONDS = 60 + +WINDOW_SIZE_SECONDS = 900 # 15 minutes + +NUM_WINDOWS = 4 + +MIN_WINDOWS_FOR_APPROACH = 3 + +R_SQUARED_APPROACHING = 0.7 +R_SQUARED_DOWNGRADE = 0.5 + +MAX_BEARING_STDDEV = 30.0 + +MAX_ETA_MINUTES = 240 + +RECONNECT_BASE_DELAY = 1.0 +RECONNECT_MAX_DELAY = 60.0 + +GPS_SAMPLE_INTERVAL = 30 + +STALE_THRESHOLD_SECONDS = 900 # 15 minutes + +DATA_DIR = '/data/dbus-lightning' + +LOGGING_CONFIG = { + 'level': 'INFO', + 'console': True, + 'include_timestamp': False, +} diff --git a/dbus-lightning/install.sh b/dbus-lightning/install.sh new file mode 100755 index 0000000..0686d23 --- /dev/null +++ b/dbus-lightning/install.sh @@ -0,0 +1,165 @@ +#!/bin/bash +# +# Installation script for Lightning Monitor on Venus OS +# +# Usage: +# chmod +x install.sh +# ./install.sh +# + +set -e + +INSTALL_DIR="/data/dbus-lightning" + +# Find velib_python +VELIB_DIR="" +if [ -d "/opt/victronenergy/velib_python" ]; then + VELIB_DIR="/opt/victronenergy/velib_python" +else + for candidate in \ + "/opt/victronenergy/dbus-systemcalc-py/ext/velib_python" \ + "/opt/victronenergy/dbus-generator/ext/velib_python" \ + "/opt/victronenergy/dbus-mqtt/ext/velib_python" \ + "/opt/victronenergy/dbus-digitalinputs/ext/velib_python" \ + "/opt/victronenergy/vrmlogger/ext/velib_python" + do + if [ -d "$candidate" ] && [ -f "$candidate/vedbus.py" ]; then + VELIB_DIR="$candidate" + break + fi + done +fi + +if [ -z "$VELIB_DIR" ]; then + VEDBUS_PATH=$(find /opt/victronenergy -name "vedbus.py" -path "*/velib_python/*" 2>/dev/null | head -1) + if [ -n "$VEDBUS_PATH" ]; then + VELIB_DIR=$(dirname "$VEDBUS_PATH") + fi +fi + +# Determine service directory +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +elif [ -L "/service" ]; then + SERVICE_DIR=$(readlink -f /service) +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "==================================================" +echo "Lightning Monitor - Installation" +echo "==================================================" + +if [ ! -d "$SERVICE_DIR" ]; then + echo "ERROR: This doesn't appear to be a Venus OS device." + echo " Service directory not found." + exit 1 +fi + +echo "Detected service directory: $SERVICE_DIR" + +if [ ! -f "$INSTALL_DIR/lightning.py" ]; then + echo "ERROR: Installation files not found in $INSTALL_DIR" + echo " Please copy all files to $INSTALL_DIR first." + exit 1 +fi +if [ ! -f "$INSTALL_DIR/service/run" ]; then + echo "ERROR: service/run not found. The package is incomplete." + exit 1 +fi + +echo "1. Making scripts executable..." +chmod +x "$INSTALL_DIR/service/run" +chmod +x "$INSTALL_DIR/service/log/run" +chmod +x "$INSTALL_DIR/lightning.py" + +echo "2. Creating velib_python symlink..." +if [ -z "$VELIB_DIR" ]; then + echo "ERROR: Could not find velib_python on this system." + exit 1 +fi +echo " Found velib_python at: $VELIB_DIR" +mkdir -p "$INSTALL_DIR/ext" +if [ -L "$INSTALL_DIR/ext/velib_python" ]; then + CURRENT_TARGET=$(readlink "$INSTALL_DIR/ext/velib_python") + if [ "$CURRENT_TARGET" != "$VELIB_DIR" ]; then + echo " Updating symlink (was: $CURRENT_TARGET)" + rm "$INSTALL_DIR/ext/velib_python" + fi +fi +if [ ! -L "$INSTALL_DIR/ext/velib_python" ]; then + ln -s "$VELIB_DIR" "$INSTALL_DIR/ext/velib_python" + echo " Symlink created: $INSTALL_DIR/ext/velib_python -> $VELIB_DIR" +else + echo " Symlink already exists" +fi + +echo "3. Creating service symlink..." +if [ -L "$SERVICE_DIR/dbus-lightning" ]; then + echo " Service link already exists, removing old link..." + rm "$SERVICE_DIR/dbus-lightning" +fi +if [ -e "$SERVICE_DIR/dbus-lightning" ]; then + rm -rf "$SERVICE_DIR/dbus-lightning" +fi +ln -s "$INSTALL_DIR/service" "$SERVICE_DIR/dbus-lightning" + +if [ -L "$SERVICE_DIR/dbus-lightning" ]; then + echo " Symlink created: $SERVICE_DIR/dbus-lightning -> $INSTALL_DIR/service" +else + echo "ERROR: Failed to create service symlink" + exit 1 +fi + +echo "4. Creating log directory..." +mkdir -p /var/log/dbus-lightning + +echo "5. Setting up rc.local for persistence..." +RC_LOCAL="/data/rc.local" +if [ ! -f "$RC_LOCAL" ]; then + echo "#!/bin/bash" > "$RC_LOCAL" + chmod +x "$RC_LOCAL" +fi + +if ! grep -q "dbus-lightning" "$RC_LOCAL"; then + echo "" >> "$RC_LOCAL" + echo "# Lightning Monitor" >> "$RC_LOCAL" + echo "if [ ! -L $SERVICE_DIR/dbus-lightning ]; then" >> "$RC_LOCAL" + echo " ln -s /data/dbus-lightning/service $SERVICE_DIR/dbus-lightning" >> "$RC_LOCAL" + echo "fi" >> "$RC_LOCAL" + echo " Added to rc.local for persistence across firmware updates" +else + echo " Already in rc.local" +fi + +echo "6. Activating service..." +sleep 2 +if command -v svstat >/dev/null 2>&1; then + if svstat "$SERVICE_DIR/dbus-lightning" 2>/dev/null | grep -q "up"; then + echo " Service is running" + else + echo " Waiting for service to start..." + sleep 3 + fi +fi + +echo "" +echo "==================================================" +echo "Installation complete!" +echo "==================================================" +echo "" + +if command -v svstat >/dev/null 2>&1; then + echo "Current status:" + svstat "$SERVICE_DIR/dbus-lightning" 2>/dev/null || echo " Service not yet detected by svscan" + echo "" +fi + +echo "To check status:" +echo " svstat $SERVICE_DIR/dbus-lightning" +echo "" +echo "To view logs:" +echo " tail -F /var/log/dbus-lightning/current | tai64nlocal" +echo "" diff --git a/dbus-lightning/lightning.py b/dbus-lightning/lightning.py new file mode 100644 index 0000000..c8f9d88 --- /dev/null +++ b/dbus-lightning/lightning.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python3 +""" +Lightning Monitor for Venus OS + +Connects to the Blitzortung community lightning detection network via +WebSocket, buffers nearby strikes within a configurable radius, detects +approaching storms, and publishes a summary to D-Bus for display in the +venus-html5-app dashboard. +""" + +import json +import logging +import os +import signal +import sys +import time + +sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'ext', 'velib_python')) +sys.path.insert(1, '/opt/victronenergy/velib_python') + +try: + from gi.repository import GLib +except ImportError: + print("ERROR: GLib not available. This script must run on Venus OS.") + sys.exit(1) + +try: + import dbus + from dbus.mainloop.glib import DBusGMainLoop + from vedbus import VeDbusService + from settingsdevice import SettingsDevice +except ImportError as e: + print(f"ERROR: Required module not available: {e}") + sys.exit(1) + +from config import ( + SERVICE_NAME, BLITZORTUNG_SERVERS, BLITZORTUNG_INIT_MSG, + STRIKE_RADIUS_MILES, STRIKE_MAX_AGE_SECONDS, + MIN_STRIKES_ACTIVE, ANALYSIS_INTERVAL_SECONDS, + WINDOW_SIZE_SECONDS, NUM_WINDOWS, MIN_WINDOWS_FOR_APPROACH, + R_SQUARED_APPROACHING, R_SQUARED_DOWNGRADE, + MAX_BEARING_STDDEV, MAX_ETA_MINUTES, + RECONNECT_BASE_DELAY, RECONNECT_MAX_DELAY, + GPS_SAMPLE_INTERVAL, STALE_THRESHOLD_SECONDS, + LOGGING_CONFIG, +) +from blitzortung_client import BlitzortungClient, STATUS_CONNECTED +from strike_buffer import StrikeBuffer +from analysis_engine import AnalysisEngine + +VERSION = '1.0.0' + +BUS_ITEM = "com.victronenergy.BusItem" +SYSTEM_SERVICE = "com.victronenergy.system" + + +def _unwrap(v): + """Minimal dbus value unwrap.""" + if v is None: + return None + if isinstance(v, (dbus.Int16, dbus.Int32, dbus.Int64, + dbus.UInt16, dbus.UInt32, dbus.UInt64, dbus.Byte)): + return int(v) + if isinstance(v, dbus.Double): + return float(v) + if isinstance(v, (dbus.String, dbus.Signature)): + return str(v) + if isinstance(v, dbus.Boolean): + return bool(v) + if isinstance(v, dbus.Array): + return [_unwrap(x) for x in v] if len(v) > 0 else None + if isinstance(v, (dbus.Dictionary, dict)): + return {k: _unwrap(x) for k, x in v.items()} + return v + + +class GpsReader: + """Read GPS position from Venus OS D-Bus.""" + + def __init__(self, bus): + self.bus = bus + self._gps_service = None + self._proxy_lat = None + self._proxy_lon = None + + def _get_proxy(self, service, path): + try: + obj = self.bus.get_object(service, path, introspect=False) + return dbus.Interface(obj, BUS_ITEM) + except dbus.exceptions.DBusException: + return None + + def _refresh_gps_service(self): + if self._gps_service: + return True + try: + proxy = self._get_proxy(SYSTEM_SERVICE, "/GpsService") + if proxy: + svc = _unwrap(proxy.GetValue()) + if svc and isinstance(svc, str): + self._gps_service = svc + self._proxy_lat = self._get_proxy(svc, "/Position/Latitude") + self._proxy_lon = self._get_proxy(svc, "/Position/Longitude") + if not self._proxy_lat: + self._proxy_lat = self._get_proxy(svc, "/Latitude") + if not self._proxy_lon: + self._proxy_lon = self._get_proxy(svc, "/Longitude") + return True + except dbus.exceptions.DBusException: + pass + return False + + def get_position(self): + """Return (lat, lon) or None if no fix.""" + if not self._refresh_gps_service(): + return None + lat, lon = None, None + try: + if self._proxy_lat: + lat = _unwrap(self._proxy_lat.GetValue()) + if self._proxy_lon: + lon = _unwrap(self._proxy_lon.GetValue()) + except dbus.exceptions.DBusException: + self._gps_service = None + return None + if (lat is not None and lon is not None and + -90 <= float(lat) <= 90 and -180 <= float(lon) <= 180): + return (float(lat), float(lon)) + return None + + +class LightningController: + """Coordinates GPS, Blitzortung client, strike buffer, analysis, and D-Bus.""" + + def __init__(self): + self._setup_logging() + self.logger = logging.getLogger('Lightning') + self.logger.info(f"Initializing Lightning Monitor v{VERSION}") + + self.bus = dbus.SystemBus() + + self._create_dbus_service() + self._setup_settings() + + self.gps = GpsReader(self.bus) + self.current_lat = None + self.current_lon = None + self.last_gps_check = 0 + + self.strike_buffer = StrikeBuffer(STRIKE_RADIUS_MILES, STRIKE_MAX_AGE_SECONDS) + + self.analysis = AnalysisEngine({ + 'window_size': WINDOW_SIZE_SECONDS, + 'num_windows': NUM_WINDOWS, + 'min_strikes_active': MIN_STRIKES_ACTIVE, + 'min_windows_approach': MIN_WINDOWS_FOR_APPROACH, + 'r2_approaching': R_SQUARED_APPROACHING, + 'r2_downgrade': R_SQUARED_DOWNGRADE, + 'max_bearing_stddev': MAX_BEARING_STDDEV, + 'max_eta_minutes': MAX_ETA_MINUTES, + }) + + self.ws_client = BlitzortungClient( + servers=BLITZORTUNG_SERVERS, + init_msg=BLITZORTUNG_INIT_MSG, + on_strike=self._on_strike, + base_delay=RECONNECT_BASE_DELAY, + max_delay=RECONNECT_MAX_DELAY, + ) + + self.last_analysis_time = 0 + self._last_status = None + self._last_disconnect_time = None + + if self.enabled: + self.ws_client.start() + + GLib.timeout_add(1000, self._main_loop) + self.logger.info("Initialized. Analysis every %ds, strike radius %.0f mi", + ANALYSIS_INTERVAL_SECONDS, STRIKE_RADIUS_MILES) + + def _setup_logging(self): + level = getattr(logging, LOGGING_CONFIG['level'], logging.INFO) + fmt = ('%(asctime)s %(levelname)s %(name)s: %(message)s' + if LOGGING_CONFIG['include_timestamp'] + else '%(levelname)s %(name)s: %(message)s') + logging.basicConfig(level=level, format=fmt, stream=sys.stdout) + + def _create_dbus_service(self): + self.logger.info(f"Creating D-Bus service: {SERVICE_NAME}") + + max_retries = 5 + retry_delay = 1.0 + for attempt in range(max_retries): + try: + self.dbus_service = VeDbusService( + SERVICE_NAME, self.bus, register=False) + break + except dbus.exceptions.NameExistsException: + if attempt < max_retries - 1: + self.logger.warning( + f"D-Bus name exists, retrying in {retry_delay}s " + f"(attempt {attempt + 1}/{max_retries})") + time.sleep(retry_delay) + retry_delay *= 2 + else: + raise + + svc = self.dbus_service + svc.add_path('/Mgmt/ProcessName', 'dbus-lightning') + svc.add_path('/Mgmt/ProcessVersion', VERSION) + svc.add_path('/Mgmt/Connection', 'local') + + svc.add_path('/DeviceInstance', 0) + svc.add_path('/ProductId', 0xA162) + svc.add_path('/ProductName', 'Lightning Monitor') + svc.add_path('/FirmwareVersion', VERSION) + svc.add_path('/Connected', 1) + + svc.add_path('/ConnectionStatus', 'disconnected') + + svc.add_path('/Active', 0) + svc.add_path('/StrikeCount15m', 0) + svc.add_path('/NearestDistance', None) + svc.add_path('/CentroidBearing', None) + svc.add_path('/CentroidDistance', None) + svc.add_path('/Cardinal', None) + svc.add_path('/Approaching', 0) + svc.add_path('/ApproachSpeed', None) + svc.add_path('/EtaMinutes', None) + svc.add_path('/Confidence', None) + svc.add_path('/LastUpdate', 0) + + svc.add_path('/Summary/Json', '') + + svc.add_path('/Settings/Enabled', 1, + writeable=True, + onchangecallback=self._on_setting_changed) + + svc.register() + self.logger.info("D-Bus service created") + + def _setup_settings(self): + self.settings = None + try: + path = '/Settings/Lightning' + settings_def = { + 'Enabled': [path + '/Enabled', 1, 0, 1], + } + self.settings = SettingsDevice( + self.bus, settings_def, + self._on_persistent_setting_changed) + if self.settings: + self._load_settings() + self.logger.info("Persistent settings initialized") + except Exception as e: + self.logger.warning(f"Could not initialize persistent settings: {e}") + self.enabled = True + + def _load_settings(self): + if not self.settings: + return + try: + self.enabled = bool(self.settings['Enabled']) + self.dbus_service['/Settings/Enabled'] = 1 if self.enabled else 0 + self.logger.info(f"Loaded settings: enabled={self.enabled}") + except Exception as e: + self.logger.warning(f"Error loading settings: {e}") + self.enabled = True + + def _on_persistent_setting_changed(self, setting, old_value, new_value): + self.logger.info(f"Persistent setting changed: {setting} = {new_value}") + self._load_settings() + + def _on_setting_changed(self, path, value): + self.logger.info(f"Setting changed: {path} = {value}") + if path == '/Settings/Enabled': + self.enabled = bool(value) + if self.settings: + try: + self.settings['Enabled'] = 1 if self.enabled else 0 + except Exception: + pass + if self.enabled: + self.ws_client.start() + else: + self.ws_client.stop() + self._clear_summary() + return True + + def _on_strike(self, lat, lon, timestamp_ms): + """Called from WebSocket thread for each decoded strike.""" + if self.current_lat is None or self.current_lon is None: + return + kept = self.strike_buffer.add( + lat, lon, timestamp_ms, self.current_lat, self.current_lon) + if kept: + self.logger.debug(f"Strike kept: ({lat:.2f}, {lon:.2f})") + + def _run_analysis(self): + """Run the analysis engine and publish results to D-Bus.""" + if self.current_lat is None or self.current_lon is None: + return + + strikes = self.strike_buffer.get_strikes() + summary = self.analysis.analyze(strikes, self.current_lat, self.current_lon) + + summary['connection_status'] = self.ws_client.status + summary['last_updated'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + + svc = self.dbus_service + svc['/Active'] = 1 if summary['active'] else 0 + svc['/StrikeCount15m'] = summary['strike_count_15m'] + svc['/NearestDistance'] = summary.get('nearest_distance') + svc['/CentroidBearing'] = summary.get('centroid_bearing') + svc['/CentroidDistance'] = summary.get('centroid_distance') + svc['/Cardinal'] = summary.get('cardinal') + svc['/Approaching'] = 1 if summary.get('approaching') else 0 + svc['/ApproachSpeed'] = summary.get('approach_speed') + svc['/EtaMinutes'] = summary.get('eta_minutes') + svc['/Confidence'] = summary.get('confidence') + svc['/LastUpdate'] = int(time.time()) + + svc['/Summary/Json'] = json.dumps(summary) + + if summary['active']: + self.logger.info( + f"Analysis: {summary['strike_count_15m']} strikes, " + f"nearest {summary.get('nearest_distance')}mi " + f"{summary.get('cardinal', '?')}, " + f"approaching={summary.get('approaching')}" + + (f", ETA {summary.get('eta_minutes')}min" + if summary.get('eta_minutes') else "")) + + def _clear_summary(self): + """Clear all summary fields on D-Bus.""" + svc = self.dbus_service + svc['/Active'] = 0 + svc['/StrikeCount15m'] = 0 + svc['/NearestDistance'] = None + svc['/CentroidBearing'] = None + svc['/CentroidDistance'] = None + svc['/Cardinal'] = None + svc['/Approaching'] = 0 + svc['/ApproachSpeed'] = None + svc['/EtaMinutes'] = None + svc['/Confidence'] = None + svc['/Summary/Json'] = '' + + def _main_loop(self): + try: + if not self.enabled: + self.dbus_service['/ConnectionStatus'] = 'disabled' + return True + + now = time.time() + + if now - self.last_gps_check >= GPS_SAMPLE_INTERVAL: + pos = self.gps.get_position() + if pos: + self.current_lat, self.current_lon = pos + self.last_gps_check = now + + current_status = self.ws_client.status + if current_status != self._last_status: + self.dbus_service['/ConnectionStatus'] = current_status + self._last_status = current_status + if current_status != STATUS_CONNECTED: + self._last_disconnect_time = now + else: + self._last_disconnect_time = None + + if (self._last_disconnect_time and + now - self._last_disconnect_time > STALE_THRESHOLD_SECONDS): + self._clear_summary() + self._last_disconnect_time = None + + if now - self.last_analysis_time >= ANALYSIS_INTERVAL_SECONDS: + self._run_analysis() + self.last_analysis_time = now + + except dbus.exceptions.DBusException as e: + self.logger.warning(f"D-Bus error: {e}") + except Exception as e: + self.logger.exception(f"Unexpected error: {e}") + + return True + + def shutdown(self): + self.ws_client.stop() + self.logger.info("Shutdown complete") + + +def main(): + DBusGMainLoop(set_as_default=True) + + print("=" * 60) + print(f"Lightning Monitor v{VERSION}") + print("=" * 60) + + mainloop = None + controller = None + + def signal_handler(signum, frame): + try: + sig_name = signal.Signals(signum).name + except ValueError: + sig_name = str(signum) + logging.info(f"Received {sig_name}, shutting down...") + if controller: + controller.shutdown() + if mainloop is not None: + mainloop.quit() + + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + + try: + controller = LightningController() + mainloop = GLib.MainLoop() + mainloop.run() + except KeyboardInterrupt: + print("\nShutdown requested") + if controller: + controller.shutdown() + except Exception as e: + logging.error(f"Fatal error: {e}", exc_info=True) + sys.exit(1) + finally: + logging.info("Service stopped") + + +if __name__ == '__main__': + main() diff --git a/dbus-lightning/service/log/run b/dbus-lightning/service/log/run new file mode 100644 index 0000000..12aea48 --- /dev/null +++ b/dbus-lightning/service/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec multilog t s99999 n8 /var/log/dbus-lightning diff --git a/dbus-lightning/service/run b/dbus-lightning/service/run new file mode 100644 index 0000000..654316a --- /dev/null +++ b/dbus-lightning/service/run @@ -0,0 +1,5 @@ +#!/bin/sh +exec 2>&1 +cd /data/dbus-lightning +export PYTHONPATH="/data/dbus-lightning/ext/velib_python:$PYTHONPATH" +exec python3 /data/dbus-lightning/lightning.py diff --git a/dbus-lightning/strike_buffer.py b/dbus-lightning/strike_buffer.py new file mode 100644 index 0000000..3c3a445 --- /dev/null +++ b/dbus-lightning/strike_buffer.py @@ -0,0 +1,108 @@ +""" +In-memory rolling buffer for lightning strikes with distance/bearing filtering. +""" + +import math +import threading +import time +from collections import deque + + +def haversine_miles(lat1, lon1, lat2, lon2): + """Great-circle distance in statute miles.""" + R = 3958.8 + dlat = math.radians(lat2 - lat1) + dlon = math.radians(lon2 - lon1) + a = (math.sin(dlat / 2) ** 2 + + math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) * + math.sin(dlon / 2) ** 2) + return R * 2 * math.asin(min(1.0, math.sqrt(a))) + + +def bearing_degrees(lat1, lon1, lat2, lon2): + """Forward azimuth from point 1 to point 2 in degrees (0=N, 90=E).""" + lat1_r, lat2_r = math.radians(lat1), math.radians(lat2) + dlon = math.radians(lon2 - lon1) + x = math.sin(dlon) * math.cos(lat2_r) + y = (math.cos(lat1_r) * math.sin(lat2_r) - + math.sin(lat1_r) * math.cos(lat2_r) * math.cos(dlon)) + return (math.degrees(math.atan2(x, y)) + 360) % 360 + + +class Strike: + __slots__ = ('timestamp', 'lat', 'lon', 'distance', 'bearing') + + def __init__(self, timestamp, lat, lon, distance, bearing): + self.timestamp = timestamp + self.lat = lat + self.lon = lon + self.distance = distance + self.bearing = bearing + + +class StrikeBuffer: + """Thread-safe rolling buffer of nearby lightning strikes.""" + + def __init__(self, radius_miles, max_age_seconds): + self.radius_miles = radius_miles + self.max_age_seconds = max_age_seconds + self._buffer = deque() + self._lock = threading.Lock() + self._total_received = 0 + self._total_kept = 0 + + def add(self, lat, lon, timestamp_ms, vessel_lat, vessel_lon): + """Filter and add a strike if within radius. Returns True if kept.""" + distance = haversine_miles(vessel_lat, vessel_lon, lat, lon) + if distance > self.radius_miles: + self._total_received += 1 + return False + + brg = bearing_degrees(vessel_lat, vessel_lon, lat, lon) + strike = Strike( + timestamp=timestamp_ms, + lat=lat, + lon=lon, + distance=distance, + bearing=brg, + ) + + with self._lock: + self._buffer.append(strike) + self._total_received += 1 + self._total_kept += 1 + self._prune() + + return True + + def _prune(self): + """Remove strikes older than max_age_seconds. Must hold lock.""" + cutoff_ms = (time.time() - self.max_age_seconds) * 1000 + while self._buffer and self._buffer[0].timestamp < cutoff_ms: + self._buffer.popleft() + + def get_strikes(self, since_seconds=None): + """Return list of Strike objects, optionally filtered by age.""" + with self._lock: + self._prune() + if since_seconds is None: + return list(self._buffer) + cutoff_ms = (time.time() - since_seconds) * 1000 + return [s for s in self._buffer if s.timestamp >= cutoff_ms] + + def clear(self): + with self._lock: + self._buffer.clear() + + @property + def count(self): + with self._lock: + return len(self._buffer) + + @property + def stats(self): + return { + 'total_received': self._total_received, + 'total_kept': self._total_kept, + 'buffered': self.count, + } diff --git a/dbus-lightning/uninstall.sh b/dbus-lightning/uninstall.sh new file mode 100755 index 0000000..dae5ba3 --- /dev/null +++ b/dbus-lightning/uninstall.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Uninstall Lightning Monitor for Venus OS + +INSTALL_DIR="/data/dbus-lightning" +SERVICE_LINK="dbus-lightning" + +# Find service directory +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "Uninstalling Lightning Monitor..." + +# Stop and remove service +if [ -L "$SERVICE_DIR/$SERVICE_LINK" ] || [ -e "$SERVICE_DIR/$SERVICE_LINK" ]; then + echo "Stopping and removing service..." + svc -d "$SERVICE_DIR/$SERVICE_LINK" 2>/dev/null || true + rm -f "$SERVICE_DIR/$SERVICE_LINK" + rm -rf "$SERVICE_DIR/$SERVICE_LINK" +fi + +echo "Service removed. Files in $INSTALL_DIR are preserved." +echo "To remove everything: rm -rf $INSTALL_DIR /var/log/dbus-lightning" diff --git a/dbus-meteoblue-forecast/.gitignore b/dbus-meteoblue-forecast/.gitignore new file mode 100644 index 0000000..26935d8 --- /dev/null +++ b/dbus-meteoblue-forecast/.gitignore @@ -0,0 +1,27 @@ +# Build artifacts +*.tar.gz +*.sha256 + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg +*.egg-info/ +dist/ +build/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Venus OS runtime (created during installation) +ext/ + +# Sensitive config (use forecast_config.example.json as template) +forecast_config.json diff --git a/dbus-meteoblue-forecast/README.md b/dbus-meteoblue-forecast/README.md new file mode 100644 index 0000000..aa6a7ba --- /dev/null +++ b/dbus-meteoblue-forecast/README.md @@ -0,0 +1,132 @@ +# Meteoblue Forecast for Venus OS + +Fetches 7-day weather forecasts from the [Meteoblue Forecast API](https://docs.meteoblue.com/en/weather-apis/forecast-api/overview) and publishes them on D-Bus for display in the venus-html5-app dashboard. + +## Features + +- 7-day forecast: wind speed, gusts, direction, precipitation, and waves +- Single API call combining basic, wind, and sea packages +- Automatic GPS position from Venus OS D-Bus +- Movement-aware refresh: 6 hours (stationary) / 3 hours (moving) +- Settings via MQTT or config file +- Persists across restarts and firmware updates + +## Prerequisites + +- Venus OS device (Cerbo GX, Venus GX, Raspberry Pi) +- GPS source connected to Venus OS +- Meteoblue API key ([register here](https://www.meteoblue.com/en/user/login/index)) + +## Installation + +1. Copy the package to the Venus OS device: + ```bash + scp -r dbus-meteoblue-forecast root@:/data/ + ``` + +2. SSH into the device and install: + ```bash + ssh root@ + chmod +x /data/dbus-meteoblue-forecast/install.sh + /data/dbus-meteoblue-forecast/install.sh + ``` + +3. Configure your API key: + ```bash + vi /data/dbus-meteoblue-forecast/forecast_config.json + ``` + Set `"api_key"` to your Meteoblue API key. + +## Configuration + +### Config file (recommended for initial setup) + +Edit `/data/dbus-meteoblue-forecast/forecast_config.json`: + +```json +{ + "api_key": "your-meteoblue-api-key" +} +``` + +### MQTT + +Publish to `W//meteoblueforecast/0/Settings/ApiKey` with `{"value": "your-key"}`. + +## D-Bus / MQTT Paths + +| Path | Type | R/W | Description | +|------|------|-----|-------------| +| `/Connected` | int | R | 1 when service is running | +| `/Status` | int | R | 0=Idle, 1=Fetching, 2=Ready, 3=Error | +| `/ErrorMessage` | string | R | Last error description | +| `/LastUpdate` | int | R | Unix timestamp of last fetch | +| `/NextUpdate` | int | R | Unix timestamp of next fetch | +| `/IsMoving` | int | R | 0=stationary, 1=moving | +| `/Forecast/Latitude` | float | R | Latitude used for forecast | +| `/Forecast/Longitude` | float | R | Longitude used for forecast | +| `/Forecast/Json` | string | R | JSON blob with forecast arrays | +| `/Settings/Enabled` | int | W | 0=disabled, 1=enabled | +| `/Settings/ApiKey` | string | W | Meteoblue API key | +| `/Settings/Units` | int | W | 0=Metric, 1=Imperial | + +### Forecast JSON Structure + +The `/Forecast/Json` path contains a JSON string with the following arrays: + +```json +{ + "ts": [1740268800000, ...], + "windspeed": [5.2, ...], + "winddirection": [180, ...], + "gust": [8.2, ...], + "precip": [0.0, ...], + "waves_height": [1.2, ...], + "waves_period": [6.0, ...], + "waves_direction": [180, ...] +} +``` + +- `ts`: Timestamps in milliseconds since epoch (UTC) +- `windspeed`: Wind speed in m/s (10m above ground) +- `winddirection`: Wind direction in degrees +- `gust`: Wind gust speed in m/s +- `precip`: Hourly precipitation in mm +- `waves_height`: Significant wave height in meters +- `waves_period`: Mean wave period in seconds +- `waves_direction`: Mean wave direction in degrees + +Wind speed in knots = windspeed * 1.94384 + +## Service Management + +```bash +# Check status +svstat /service/dbus-meteoblue-forecast + +# View logs +tail -F /var/log/dbus-meteoblue-forecast/current | tai64nlocal + +# Restart service +svc -t /service/dbus-meteoblue-forecast + +# Stop service +svc -d /service/dbus-meteoblue-forecast +``` + +## Uninstall + +```bash +chmod +x /data/dbus-meteoblue-forecast/uninstall.sh +/data/dbus-meteoblue-forecast/uninstall.sh +``` + +## API Details + +Uses the [Meteoblue Forecast API](https://docs.meteoblue.com/en/weather-apis/forecast-api/overview): + +- **Endpoint**: `GET https://my.meteoblue.com/packages/basic-1h_wind-1h_sea-1h` +- **basic-1h**: windspeed, winddirection, precipitation (hourly, 7-day) +- **wind-1h**: windgusts (hourly, 7-day) +- **sea-1h**: wave height, period, direction (hourly, valid 20km+ offshore) +- **Rate limit**: 500 calls/minute (default) diff --git a/dbus-meteoblue-forecast/build-package.sh b/dbus-meteoblue-forecast/build-package.sh new file mode 100755 index 0000000..288728d --- /dev/null +++ b/dbus-meteoblue-forecast/build-package.sh @@ -0,0 +1,165 @@ +#!/bin/bash +# +# Build script for Meteoblue Forecast Venus OS package +# +# Creates a tar.gz package that can be: +# 1. Copied to a Venus OS device (Cerbo GX, Venus GX, etc.) +# 2. Untarred to /data/ +# 3. Installed by running install.sh +# +# Usage: +# ./build-package.sh # Creates package with default name +# ./build-package.sh --version 2.0.0 # Creates package with version in name +# ./build-package.sh --output /path/ # Specify output directory +# +# Installation on Venus OS: +# scp dbus-meteoblue-forecast-*.tar.gz root@:/data/ +# ssh root@ +# cd /data && tar -xzf dbus-meteoblue-forecast-*.tar.gz +# bash /data/dbus-meteoblue-forecast/install.sh +# Edit /data/dbus-meteoblue-forecast/forecast_config.json with your API key +# + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +VERSION="2.0.0" +OUTPUT_DIR="$SCRIPT_DIR" +PACKAGE_NAME="dbus-meteoblue-forecast" + +while [[ $# -gt 0 ]]; do + case $1 in + --version|-v) + VERSION="$2" + shift 2 + ;; + --output|-o) + OUTPUT_DIR="$2" + shift 2 + ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " -v, --version VERSION Set package version (default: 2.0.0)" + echo " -o, --output PATH Output directory (default: script directory)" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +BUILD_DATE=$(date -u +"%Y-%m-%d %H:%M:%S UTC") +BUILD_TIMESTAMP=$(date +%Y%m%d%H%M%S) + +BUILD_DIR=$(mktemp -d) +PACKAGE_DIR="$BUILD_DIR/$PACKAGE_NAME" + +echo "==================================================" +echo "Building $PACKAGE_NAME package" +echo "==================================================" +echo "Version: $VERSION" +echo "Build date: $BUILD_DATE" +echo "Source: $SCRIPT_DIR" +echo "Output: $OUTPUT_DIR" +echo "" + +echo "1. Creating package structure..." +mkdir -p "$PACKAGE_DIR" +mkdir -p "$PACKAGE_DIR/service/log" + +[ "$(uname)" = "Darwin" ] && export COPYFILE_DISABLE=1 + +echo "2. Copying application files..." +cp "$SCRIPT_DIR/meteoblue_forecast.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/config.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/forecast_config.json" "$PACKAGE_DIR/" + +echo "3. Copying service files..." +cp "$SCRIPT_DIR/service/run" "$PACKAGE_DIR/service/" +cp "$SCRIPT_DIR/service/log/run" "$PACKAGE_DIR/service/log/" + +echo "4. Copying installation scripts..." +cp "$SCRIPT_DIR/install.sh" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/uninstall.sh" "$PACKAGE_DIR/" + +echo "5. Copying documentation..." +if [ -f "$SCRIPT_DIR/README.md" ]; then + cp "$SCRIPT_DIR/README.md" "$PACKAGE_DIR/" +fi + +echo "6. Creating version info..." +cat > "$PACKAGE_DIR/VERSION" << EOF +Package: $PACKAGE_NAME +Version: $VERSION +Build Date: $BUILD_DATE +Build Timestamp: $BUILD_TIMESTAMP + +Installation: + 1. Copy to Venus OS: scp $PACKAGE_NAME-$VERSION.tar.gz root@:/data/ + 2. SSH to device: ssh root@ + 3. Extract: cd /data && tar -xzf $PACKAGE_NAME-$VERSION.tar.gz + 4. Install: bash /data/$PACKAGE_NAME/install.sh + 5. Configure: edit /data/$PACKAGE_NAME/forecast_config.json with your Meteoblue API key +EOF + +echo "7. Setting permissions..." +chmod +x "$PACKAGE_DIR/meteoblue_forecast.py" +chmod +x "$PACKAGE_DIR/install.sh" +chmod +x "$PACKAGE_DIR/uninstall.sh" +chmod +x "$PACKAGE_DIR/service/run" +chmod +x "$PACKAGE_DIR/service/log/run" + +mkdir -p "$OUTPUT_DIR" + +TARBALL_NAME="$PACKAGE_NAME-$VERSION.tar.gz" +OUTPUT_DIR_ABS="$(cd "$OUTPUT_DIR" && pwd)" +TARBALL_PATH="$OUTPUT_DIR_ABS/$TARBALL_NAME" + +echo "8. Creating package archive..." +cd "$BUILD_DIR" +if [ "$(uname)" = "Darwin" ]; then + if command -v xattr >/dev/null 2>&1; then + xattr -cr "$PACKAGE_NAME" + fi +fi +tar --format=ustar -czf "$TARBALL_PATH" "$PACKAGE_NAME" + +if command -v sha256sum >/dev/null 2>&1; then + CHECKSUM=$(sha256sum "$TARBALL_PATH" | cut -d' ' -f1) +else + CHECKSUM=$(shasum -a 256 "$TARBALL_PATH" | cut -d' ' -f1) +fi +echo "$CHECKSUM $TARBALL_NAME" > "$OUTPUT_DIR_ABS/$TARBALL_NAME.sha256" + +echo "9. Cleaning up..." +rm -rf "$BUILD_DIR" + +if [ "$(uname)" = "Darwin" ]; then + FILE_SIZE=$(du -h "$TARBALL_PATH" | cut -f1) +else + FILE_SIZE=$(du -h "$TARBALL_PATH" | cut -f1) +fi + +echo "" +echo "==================================================" +echo "Build complete!" +echo "==================================================" +echo "" +echo "Package: $TARBALL_PATH" +echo "Size: $FILE_SIZE" +echo "SHA256: $CHECKSUM" +echo "" +echo "Installation on Venus OS:" +echo " scp $TARBALL_PATH root@:/data/" +echo " ssh root@" +echo " cd /data" +echo " tar -xzf $TARBALL_NAME" +echo " bash /data/$PACKAGE_NAME/install.sh" +echo " vi /data/$PACKAGE_NAME/forecast_config.json # set your API key" +echo "" diff --git a/dbus-meteoblue-forecast/config.py b/dbus-meteoblue-forecast/config.py new file mode 100644 index 0000000..ec4e6a4 --- /dev/null +++ b/dbus-meteoblue-forecast/config.py @@ -0,0 +1,66 @@ +""" +Configuration for Meteoblue Forecast Venus OS service. + +All tunable parameters in one place for easy adjustment. +""" + +# ============================================================================= +# D-BUS SERVICE CONFIGURATION +# ============================================================================= + +SERVICE_NAME = 'com.victronenergy.meteoblueforecast' + +# ============================================================================= +# METEOBLUE FORECAST API CONFIGURATION +# ============================================================================= + +API_URL = 'https://my.meteoblue.com/packages/basic-1h_wind-1h_sea-1h_sunmoon' + +API_PARAMS = { + 'windspeed': 'ms-1', + 'forecast_days': '7', + 'timeformat': 'timestamp_ms_utc', + 'tz': 'utc', +} + +# ============================================================================= +# TIMING CONFIGURATION +# ============================================================================= + +# Forecast refresh interval when vessel is stationary (seconds) +REFRESH_INTERVAL_STATIONARY = 6 * 3600 # 6 hours + +# Forecast refresh interval when vessel is moving (seconds) +REFRESH_INTERVAL_MOVING = 3 * 3600 # 3 hours + +# How often to sample GPS for movement detection (seconds) +GPS_SAMPLE_INTERVAL = 60 + +# How many GPS positions to keep for movement detection +GPS_HISTORY_SIZE = 4 + +# GPS sampling interval for history (seconds) -- sample every 15 minutes +GPS_HISTORY_SAMPLE_INTERVAL = 900 + +# Movement threshold in meters (~0.5 nautical miles) +MOVEMENT_THRESHOLD_METERS = 926.0 + +# Forecast horizon in hours (7 days) +FORECAST_HOURS = 168 + +# ============================================================================= +# PATH CONFIGURATION +# ============================================================================= + +DATA_DIR = '/data/dbus-meteoblue-forecast' +CONFIG_FILE = DATA_DIR + '/forecast_config.json' + +# ============================================================================= +# LOGGING CONFIGURATION +# ============================================================================= + +LOGGING_CONFIG = { + 'level': 'INFO', + 'console': True, + 'include_timestamp': False, +} diff --git a/dbus-meteoblue-forecast/forecast_config.example.json b/dbus-meteoblue-forecast/forecast_config.example.json new file mode 100644 index 0000000..338d6ff --- /dev/null +++ b/dbus-meteoblue-forecast/forecast_config.example.json @@ -0,0 +1,3 @@ +{ + "api_key": "YOUR_METEOBLUE_API_KEY" +} diff --git a/dbus-meteoblue-forecast/install.sh b/dbus-meteoblue-forecast/install.sh new file mode 100644 index 0000000..e64051f --- /dev/null +++ b/dbus-meteoblue-forecast/install.sh @@ -0,0 +1,171 @@ +#!/bin/bash +# +# Installation script for Meteoblue Forecast on Venus OS +# +# Run this on the Venus OS device after copying files to /data/dbus-meteoblue-forecast/ +# +# Usage: +# chmod +x install.sh +# ./install.sh +# + +set -e + +INSTALL_DIR="/data/dbus-meteoblue-forecast" + +# Find velib_python +VELIB_DIR="" +if [ -d "/opt/victronenergy/velib_python" ]; then + VELIB_DIR="/opt/victronenergy/velib_python" +else + for candidate in \ + "/opt/victronenergy/dbus-systemcalc-py/ext/velib_python" \ + "/opt/victronenergy/dbus-generator/ext/velib_python" \ + "/opt/victronenergy/dbus-mqtt/ext/velib_python" \ + "/opt/victronenergy/dbus-digitalinputs/ext/velib_python" \ + "/opt/victronenergy/vrmlogger/ext/velib_python" + do + if [ -d "$candidate" ] && [ -f "$candidate/vedbus.py" ]; then + VELIB_DIR="$candidate" + break + fi + done +fi + +if [ -z "$VELIB_DIR" ]; then + VEDBUS_PATH=$(find /opt/victronenergy -name "vedbus.py" -path "*/velib_python/*" 2>/dev/null | head -1) + if [ -n "$VEDBUS_PATH" ]; then + VELIB_DIR=$(dirname "$VEDBUS_PATH") + fi +fi + +# Determine service directory +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +elif [ -L "/service" ]; then + SERVICE_DIR=$(readlink -f /service) +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "==================================================" +echo "Meteoblue Forecast - Installation" +echo "==================================================" + +if [ ! -d "$SERVICE_DIR" ]; then + echo "ERROR: This doesn't appear to be a Venus OS device." + echo " Service directory not found." + exit 1 +fi + +echo "Detected service directory: $SERVICE_DIR" + +if [ ! -f "$INSTALL_DIR/meteoblue_forecast.py" ]; then + echo "ERROR: Installation files not found in $INSTALL_DIR" + echo " Please copy all files to $INSTALL_DIR first." + exit 1 +fi +if [ ! -f "$INSTALL_DIR/service/run" ]; then + echo "ERROR: service/run not found. The package is incomplete." + exit 1 +fi + +echo "1. Making scripts executable..." +chmod +x "$INSTALL_DIR/service/run" +chmod +x "$INSTALL_DIR/service/log/run" +chmod +x "$INSTALL_DIR/meteoblue_forecast.py" + +echo "2. Creating velib_python symlink..." +if [ -z "$VELIB_DIR" ]; then + echo "ERROR: Could not find velib_python on this system." + exit 1 +fi +echo " Found velib_python at: $VELIB_DIR" +mkdir -p "$INSTALL_DIR/ext" +if [ -L "$INSTALL_DIR/ext/velib_python" ]; then + CURRENT_TARGET=$(readlink "$INSTALL_DIR/ext/velib_python") + if [ "$CURRENT_TARGET" != "$VELIB_DIR" ]; then + echo " Updating symlink (was: $CURRENT_TARGET)" + rm "$INSTALL_DIR/ext/velib_python" + fi +fi +if [ ! -L "$INSTALL_DIR/ext/velib_python" ]; then + ln -s "$VELIB_DIR" "$INSTALL_DIR/ext/velib_python" + echo " Symlink created: $INSTALL_DIR/ext/velib_python -> $VELIB_DIR" +else + echo " Symlink already exists" +fi + +echo "3. Creating service symlink..." +if [ -L "$SERVICE_DIR/dbus-meteoblue-forecast" ]; then + echo " Service link already exists, removing old link..." + rm "$SERVICE_DIR/dbus-meteoblue-forecast" +fi +if [ -e "$SERVICE_DIR/dbus-meteoblue-forecast" ]; then + rm -rf "$SERVICE_DIR/dbus-meteoblue-forecast" +fi +ln -s "$INSTALL_DIR/service" "$SERVICE_DIR/dbus-meteoblue-forecast" + +if [ -L "$SERVICE_DIR/dbus-meteoblue-forecast" ]; then + echo " Symlink created: $SERVICE_DIR/dbus-meteoblue-forecast -> $INSTALL_DIR/service" +else + echo "ERROR: Failed to create service symlink" + exit 1 +fi + +echo "4. Creating log directory..." +mkdir -p /var/log/dbus-meteoblue-forecast + +echo "5. Setting up rc.local for persistence..." +RC_LOCAL="/data/rc.local" +if [ ! -f "$RC_LOCAL" ]; then + echo "#!/bin/bash" > "$RC_LOCAL" + chmod +x "$RC_LOCAL" +fi + +if ! grep -q "dbus-meteoblue-forecast" "$RC_LOCAL"; then + echo "" >> "$RC_LOCAL" + echo "# Meteoblue Forecast" >> "$RC_LOCAL" + echo "if [ ! -L $SERVICE_DIR/dbus-meteoblue-forecast ]; then" >> "$RC_LOCAL" + echo " ln -s /data/dbus-meteoblue-forecast/service $SERVICE_DIR/dbus-meteoblue-forecast" >> "$RC_LOCAL" + echo "fi" >> "$RC_LOCAL" + echo " Added to rc.local for persistence across firmware updates" +else + echo " Already in rc.local" +fi + +echo "6. Activating service..." +sleep 2 +if command -v svstat >/dev/null 2>&1; then + if svstat "$SERVICE_DIR/dbus-meteoblue-forecast" 2>/dev/null | grep -q "up"; then + echo " Service is running" + else + echo " Waiting for service to start..." + sleep 3 + fi +fi + +echo "" +echo "==================================================" +echo "Installation complete!" +echo "==================================================" +echo "" + +if command -v svstat >/dev/null 2>&1; then + echo "Current status:" + svstat "$SERVICE_DIR/dbus-meteoblue-forecast" 2>/dev/null || echo " Service not yet detected by svscan" + echo "" +fi + +echo "IMPORTANT: Configure your Meteoblue API key:" +echo " 1. Edit $INSTALL_DIR/forecast_config.json" +echo " 2. Or use MQTT: W//meteoblueforecast/0/Settings/ApiKey" +echo "" +echo "To check status:" +echo " svstat $SERVICE_DIR/dbus-meteoblue-forecast" +echo "" +echo "To view logs:" +echo " tail -F /var/log/dbus-meteoblue-forecast/current | tai64nlocal" +echo "" diff --git a/dbus-meteoblue-forecast/meteoblue_forecast.py b/dbus-meteoblue-forecast/meteoblue_forecast.py new file mode 100644 index 0000000..c687b05 --- /dev/null +++ b/dbus-meteoblue-forecast/meteoblue_forecast.py @@ -0,0 +1,732 @@ +#!/usr/bin/env python3 +""" +Meteoblue Forecast for Venus OS + +Fetches 7-day weather forecasts from the Meteoblue Forecast API and +publishes them on D-Bus for display in the venus-html5-app dashboard. + +Uses a single API call combining four packages: +- basic-1h: windspeed, winddirection, precipitation, temperature +- wind-1h: gust (wind gusts), windspeed_80m, winddirection_80m +- sea-1h: wave height, period, direction +- sunmoon: sunrise/sunset, moonrise/moonset, moon phase + +GPS position is read from Venus OS D-Bus. Movement detection determines +the refresh interval: +- Stationary vessel: every 6 hours +- Moving vessel: every 3 hours +""" + +import json +import logging +import math +import os +import signal +import sys +import time +import threading +from collections import deque +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'ext', 'velib_python')) +sys.path.insert(1, '/opt/victronenergy/velib_python') + +try: + from gi.repository import GLib +except ImportError: + print("ERROR: GLib not available. This script must run on Venus OS.") + sys.exit(1) + +try: + import dbus + from dbus.mainloop.glib import DBusGMainLoop + from vedbus import VeDbusService + from settingsdevice import SettingsDevice +except ImportError as e: + print(f"ERROR: Required module not available: {e}") + print("This script must run on Venus OS.") + sys.exit(1) + +from config import ( + SERVICE_NAME, API_URL, API_PARAMS, + REFRESH_INTERVAL_STATIONARY, REFRESH_INTERVAL_MOVING, + GPS_SAMPLE_INTERVAL, GPS_HISTORY_SIZE, GPS_HISTORY_SAMPLE_INTERVAL, + MOVEMENT_THRESHOLD_METERS, FORECAST_HOURS, + CONFIG_FILE, LOGGING_CONFIG, +) + +VERSION = '2.1.0' + +BUS_ITEM = "com.victronenergy.BusItem" +SYSTEM_SERVICE = "com.victronenergy.system" + + +def _unwrap(v): + """Minimal dbus value unwrap.""" + if v is None: + return None + if isinstance(v, (dbus.Int16, dbus.Int32, dbus.Int64, + dbus.UInt16, dbus.UInt32, dbus.UInt64, dbus.Byte)): + return int(v) + if isinstance(v, dbus.Double): + return float(v) + if isinstance(v, (dbus.String, dbus.Signature)): + return str(v) + if isinstance(v, dbus.Boolean): + return bool(v) + if isinstance(v, dbus.Array): + return [_unwrap(x) for x in v] if len(v) > 0 else None + if isinstance(v, (dbus.Dictionary, dict)): + return {k: _unwrap(x) for k, x in v.items()} + return v + + +def haversine_distance(lat1, lon1, lat2, lon2): + """Great-circle distance in meters between two GPS coordinates.""" + R = 6371000 + phi1, phi2 = math.radians(lat1), math.radians(lat2) + dphi = math.radians(lat2 - lat1) + dlam = math.radians(lon2 - lon1) + a = (math.sin(dphi / 2) ** 2 + + math.cos(phi1) * math.cos(phi2) * math.sin(dlam / 2) ** 2) + return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) + + +class GpsReader: + """Read GPS position from Venus OS D-Bus.""" + + def __init__(self, bus): + self.bus = bus + self._gps_service = None + self._proxy_lat = None + self._proxy_lon = None + + def _get_proxy(self, service, path): + try: + obj = self.bus.get_object(service, path, introspect=False) + return dbus.Interface(obj, BUS_ITEM) + except dbus.exceptions.DBusException: + return None + + def _refresh_gps_service(self): + if self._gps_service: + return True + try: + proxy = self._get_proxy(SYSTEM_SERVICE, "/GpsService") + if proxy: + svc = _unwrap(proxy.GetValue()) + if svc and isinstance(svc, str): + self._gps_service = svc + self._proxy_lat = self._get_proxy(svc, "/Position/Latitude") + self._proxy_lon = self._get_proxy(svc, "/Position/Longitude") + if not self._proxy_lat: + self._proxy_lat = self._get_proxy(svc, "/Latitude") + if not self._proxy_lon: + self._proxy_lon = self._get_proxy(svc, "/Longitude") + return True + except dbus.exceptions.DBusException: + pass + return False + + def get_position(self): + """Return (lat, lon) or None if no fix.""" + if not self._refresh_gps_service(): + return None + lat, lon = None, None + try: + if self._proxy_lat: + lat = _unwrap(self._proxy_lat.GetValue()) + if self._proxy_lon: + lon = _unwrap(self._proxy_lon.GetValue()) + except dbus.exceptions.DBusException: + self._gps_service = None + return None + if (lat is not None and lon is not None and + -90 <= float(lat) <= 90 and -180 <= float(lon) <= 180): + return (float(lat), float(lon)) + return None + + +class MovementDetector: + """Determines whether the vessel is moving based on GPS history.""" + + def __init__(self, history_size, threshold_meters): + self.history_size = history_size + self.threshold_meters = threshold_meters + self.positions = deque(maxlen=history_size) + self.last_sample_time = 0 + + def add_position(self, lat, lon, now=None): + if now is None: + now = time.time() + self.positions.append((now, lat, lon)) + self.last_sample_time = now + + def is_moving(self, current_lat, current_lon): + """Check if vessel has moved beyond threshold from any stored position.""" + if not self.positions: + return False + for _, hist_lat, hist_lon in self.positions: + dist = haversine_distance(hist_lat, hist_lon, current_lat, current_lon) + if dist >= self.threshold_meters: + return True + return False + + +class MeteoblueApiClient: + """Fetches forecast data from the Meteoblue Forecast API.""" + + def __init__(self, logger): + self.logger = logger + + def fetch_forecast(self, api_key, lat, lon): + """Fetch combined basic + wind + sea + sunmoon forecast. Returns normalized dict or None.""" + if not api_key: + self.logger.warning("No API key configured") + return None + + rounded_lat = round(lat, 2) + rounded_lon = round(lon, 2) + + params = dict(API_PARAMS) + params['lat'] = str(rounded_lat) + params['lon'] = str(rounded_lon) + params['apikey'] = api_key + + url = f"{API_URL}?{urlencode(params)}" + + try: + request = Request(url, method='GET') + with urlopen(request, timeout=30) as response: + body = response.read().decode('utf-8') + data = json.loads(body) + return self._normalize(data) + except HTTPError as e: + body = "" + try: + body = e.read().decode('utf-8') + except Exception: + pass + self.logger.error(f"Meteoblue HTTP {e.code}: {e.reason} {body}") + return None + except (URLError, Exception) as e: + self.logger.error(f"Meteoblue request error: {e}") + return None + + def _normalize(self, data): + """Extract and normalize meteoblue response into forecast JSON. + + Meteoblue returns data grouped by time resolution, e.g. data_1h for + hourly packages. When combining basic-1h, wind-1h, sea-1h, and + sunmoon, each package may appear in its own section or be merged. + Hourly variables are searched across all data_1h sections. The + sunmoon package produces daily data in a data_day section with + rise/set times and moon phase information. + """ + hourly_sections = [] + for key in sorted(data.keys()): + if key.startswith('data_1h') and isinstance(data[key], dict): + hourly_sections.append((key, data[key])) + + if not hourly_sections: + self.logger.warning("No hourly data sections in response") + return None + + self.logger.info( + f"Response sections: {[k for k in sorted(data.keys()) if k.startswith('data_')]}") + + hourly = hourly_sections[0][1] + ts = hourly.get('time', []) + if not ts: + self.logger.warning("No hourly timestamps in response") + return None + + now_ms = int(time.time() * 1000) + horizon_ms = now_ms + FORECAST_HOURS * 3600 * 1000 + + if isinstance(ts[0], (int, float)) and ts[0] > 1e12: + indices = [i for i, t in enumerate(ts) if t <= horizon_ms] + else: + indices = list(range(len(ts))) + if not indices: + indices = list(range(len(ts))) + + def pick(arr): + if not arr: + return [] + return [arr[i] if i < len(arr) else None for i in indices] + + def find_var(key): + """Search all hourly sections for a variable, skipping sea sections + which may have different timestamps.""" + for section_key, section in hourly_sections: + if 'sea' in section_key: + continue + arr = section.get(key, []) + if arr: + return arr + return [] + + result = { + 'ts': pick(ts), + 'windspeed': pick(find_var('windspeed')), + 'winddirection': pick(find_var('winddirection')), + 'gust': pick(find_var('gust')), + 'precip': pick(find_var('precipitation')), + 'temperature': pick(find_var('temperature')), + } + + def find_sea_section(): + """Find the sea data section, which may have different timestamps.""" + for section_key, section in hourly_sections: + if 'sea' in section_key: + return section + if section.get('sigwaveheight'): + return section + return hourly + + sea = find_sea_section() + sea_ts = sea.get('time', ts) + + if sea is not hourly and sea_ts: + if isinstance(sea_ts[0], (int, float)) and sea_ts[0] > 1e12: + sea_indices = [i for i, t in enumerate(sea_ts) if t <= horizon_ms] + else: + sea_indices = list(range(len(sea_ts))) + if not sea_indices: + sea_indices = list(range(len(sea_ts))) + else: + sea_indices = indices + + def pick_sea(key): + arr = sea.get(key, []) + if not arr: + return [] + return [arr[i] if i < len(arr) else None for i in sea_indices] + + result['waves_height'] = pick_sea('sigwaveheight') + result['waves_period'] = pick_sea('meanwaveperiod') + result['waves_direction'] = pick_sea('meanwavedirection') + + sunmoon = self._extract_sunmoon(data) + if sunmoon: + result['sunmoon'] = sunmoon + + gust_count = sum(1 for v in result['gust'] if v is not None) + ts_count = len(result['ts']) + sunmoon_days = len(sunmoon.get('time', [])) if sunmoon else 0 + self.logger.info( + f"Normalized forecast: {ts_count} timestamps, " + f"{gust_count} gust values, {sunmoon_days} sunmoon days") + + return result + + def _extract_sunmoon(self, data): + """Extract sun/moon daily data from the sunmoon package. + + The sunmoon package returns a data_day section with rise/set times + as "hh:mm" strings and moon phase information. Special values: + "---" means the body does not rise or set that day; "00:00"/"24:00" + means it is visible the entire day. + + Multiple data_day sections may exist (e.g. data_day from basic, + data_day_sunmoon from sunmoon package), so we merge across all of them. + """ + SUNMOON_KEYS = [ + 'sunrise', 'sunset', + 'moonrise', 'moonset', + 'moonphaseangle', 'moonage', 'moonphasename', + 'moonphasetransit', 'moonillumination', + ] + + daily_sections = [] + for key in sorted(data.keys()): + if key.startswith('data_day') and isinstance(data[key], dict): + section = data[key] + if any(section.get(k) for k in SUNMOON_KEYS): + daily_sections.append(section) + + if not daily_sections: + self.logger.debug("No sun/moon daily data in response") + return None + + result = {} + for section in daily_sections: + daily_time = section.get('time', []) + if daily_time and 'time' not in result: + result['time'] = list(daily_time) + + for var_key in SUNMOON_KEYS: + if var_key in result: + continue + arr = section.get(var_key) + if arr: + result[var_key] = list(arr) + + return result if len(result) > 1 else None + + +class MeteoblueForecastController: + """Coordinates GPS, movement detection, API calls, and D-Bus publishing.""" + + def __init__(self): + self._setup_logging() + self.logger = logging.getLogger('MeteoblueForecast') + self.logger.info(f"Initializing Meteoblue Forecast v{VERSION}") + + self.bus = dbus.SystemBus() + + self._create_dbus_service() + self._setup_settings() + + self.gps = GpsReader(self.bus) + self.movement = MovementDetector( + GPS_HISTORY_SIZE, MOVEMENT_THRESHOLD_METERS) + self.api_client = MeteoblueApiClient(self.logger) + + self.last_fetch_time = 0 + self.last_gps_check = 0 + self.last_gps_history_sample = 0 + self.current_lat = None + self.current_lon = None + self._fetch_in_progress = False + + GLib.timeout_add(1000, self._main_loop) + self.logger.info( + "Initialized. Checking GPS every 60s, " + f"forecast refresh: {REFRESH_INTERVAL_STATIONARY}s " + f"(stationary) / {REFRESH_INTERVAL_MOVING}s (moving)") + + def _setup_logging(self): + level = getattr(logging, LOGGING_CONFIG['level'], logging.INFO) + fmt = ('%(asctime)s %(levelname)s %(name)s: %(message)s' + if LOGGING_CONFIG['include_timestamp'] + else '%(levelname)s %(name)s: %(message)s') + logging.basicConfig(level=level, format=fmt, stream=sys.stdout) + + def _create_dbus_service(self): + self.logger.info(f"Creating D-Bus service: {SERVICE_NAME}") + + max_retries = 5 + retry_delay = 1.0 + for attempt in range(max_retries): + try: + self.dbus_service = VeDbusService( + SERVICE_NAME, self.bus, register=False) + break + except dbus.exceptions.NameExistsException: + if attempt < max_retries - 1: + self.logger.warning( + f"D-Bus name exists, retrying in {retry_delay}s " + f"(attempt {attempt + 1}/{max_retries})") + time.sleep(retry_delay) + retry_delay *= 2 + else: + raise + + self.dbus_service.add_path('/Mgmt/ProcessName', 'dbus-meteoblue-forecast') + self.dbus_service.add_path('/Mgmt/ProcessVersion', VERSION) + self.dbus_service.add_path('/Mgmt/Connection', 'local') + + self.dbus_service.add_path('/DeviceInstance', 0) + self.dbus_service.add_path('/ProductId', 0xA161) + self.dbus_service.add_path('/ProductName', 'Meteoblue Forecast') + self.dbus_service.add_path('/FirmwareVersion', VERSION) + self.dbus_service.add_path('/Connected', 1) + + def _status_text(p, v): + labels = {0: 'Idle', 1: 'Fetching', 2: 'Ready', 3: 'Error'} + return labels.get(v, 'Unknown') if v is not None else 'Unknown' + + self.dbus_service.add_path('/Status', 0, gettextcallback=_status_text) + self.dbus_service.add_path('/ErrorMessage', '') + self.dbus_service.add_path('/LastUpdate', 0, + gettextcallback=self._time_ago_text) + self.dbus_service.add_path('/NextUpdate', 0, + gettextcallback=self._time_until_text) + self.dbus_service.add_path('/IsMoving', 0) + + self.dbus_service.add_path('/Forecast/Latitude', None) + self.dbus_service.add_path('/Forecast/Longitude', None) + self.dbus_service.add_path('/Forecast/Json', '') + + self.dbus_service.add_path('/Settings/Enabled', 1, + writeable=True, + onchangecallback=self._on_setting_changed) + self.dbus_service.add_path('/Settings/ApiKey', '', + writeable=True, + onchangecallback=self._on_setting_changed) + self.dbus_service.add_path('/Settings/Units', 0, + writeable=True, + onchangecallback=self._on_setting_changed) + + self.dbus_service.register() + self.logger.info("D-Bus service created") + + def _setup_settings(self): + self.settings = None + try: + path = '/Settings/MeteoblueForecast' + settings_def = { + 'Enabled': [path + '/Enabled', 1, 0, 1], + 'ApiKey': [path + '/ApiKey', '', 0, 0], + 'Units': [path + '/Units', 0, 0, 1], + } + self.settings = SettingsDevice( + self.bus, settings_def, + self._on_persistent_setting_changed) + if self.settings: + self._load_settings() + self.logger.info("Persistent settings initialized") + except Exception as e: + self.logger.warning( + f"Could not initialize persistent settings: {e}") + self._set_defaults() + self._load_config_file() + + def _set_defaults(self): + self.enabled = True + self.api_key = '' + self.units = 0 + + def _load_config_file(self): + """Load forecast_config.json if it exists.""" + try: + if not os.path.exists(CONFIG_FILE): + return + with open(CONFIG_FILE, 'r') as f: + data = json.load(f) + self.logger.info(f"Loading config from {CONFIG_FILE}") + + field_map = { + 'api_key': ('ApiKey', 'api_key', str), + 'apiKey': ('ApiKey', 'api_key', str), + 'key': ('ApiKey', 'api_key', str), + } + + for json_key, value in data.items(): + if json_key not in field_map: + continue + setting_name, attr_name, cast = field_map[json_key] + try: + typed_value = cast(value) if value is not None else None + if typed_value is None: + continue + setattr(self, attr_name, typed_value) + self._save_setting(setting_name, typed_value) + dbus_path = f'/Settings/{setting_name}' + self.dbus_service[dbus_path] = typed_value + except (ValueError, TypeError): + self.logger.warning( + f"Invalid value for {json_key}: {value}") + + self.logger.info( + f"Config applied: api_key={'(set)' if self.api_key else '(empty)'}") + except json.JSONDecodeError as e: + self.logger.error(f"Invalid JSON in {CONFIG_FILE}: {e}") + except Exception as e: + self.logger.error(f"Error reading {CONFIG_FILE}: {e}") + + def _load_settings(self): + if not self.settings: + return + try: + self.enabled = bool(self.settings['Enabled']) + self.api_key = str(self.settings['ApiKey'] or '') + self.units = int(self.settings['Units'] or 0) + + self.dbus_service['/Settings/Enabled'] = 1 if self.enabled else 0 + self.dbus_service['/Settings/ApiKey'] = self.api_key + self.dbus_service['/Settings/Units'] = self.units + + self.logger.info( + f"Loaded settings: api_key={'(set)' if self.api_key else '(empty)'}") + except Exception as e: + self.logger.warning(f"Error loading settings: {e}") + self._set_defaults() + + def _on_persistent_setting_changed(self, setting, old_value, new_value): + self.logger.info(f"Persistent setting changed: {setting} = {new_value}") + self._load_settings() + + def _on_setting_changed(self, path, value): + self.logger.info(f"Setting changed: {path} = {value}") + if path == '/Settings/Enabled': + self.enabled = bool(value) + self._save_setting('Enabled', 1 if self.enabled else 0) + elif path == '/Settings/ApiKey': + self.api_key = str(value) if value else '' + self._save_setting('ApiKey', self.api_key) + elif path == '/Settings/Units': + self.units = int(value) if value is not None else 0 + self._save_setting('Units', self.units) + return True + + def _save_setting(self, name, value): + if self.settings: + try: + self.settings[name] = value + except Exception as e: + self.logger.warning(f"Failed to save setting {name}: {e}") + + def _time_ago_text(self, path, value): + try: + if value is None or (isinstance(value, (int, float)) and value <= 0): + return "Never" + diff = time.time() - float(value) + if diff < 60: + return "Just now" + if diff < 3600: + return f"{int(diff / 60)}m ago" + h = int(diff / 3600) + m = int((diff % 3600) / 60) + return f"{h}h {m}m ago" + except (TypeError, ValueError): + return "Never" + + def _time_until_text(self, path, value): + try: + if value is None or (isinstance(value, (int, float)) and value <= 0): + return "--" + diff = float(value) - time.time() + if diff <= 0: + return "Now" + if diff < 3600: + return f"in {int(diff / 60)}m" + h = int(diff / 3600) + m = int((diff % 3600) / 60) + return f"in {h}h {m}m" + except (TypeError, ValueError): + return "--" + + def _do_fetch(self, lat, lon): + """Run in a background thread to avoid blocking D-Bus.""" + try: + self.logger.info(f"Fetching forecast for {lat:.2f}, {lon:.2f}") + self.dbus_service['/Status'] = 1 + + forecast = self.api_client.fetch_forecast(self.api_key, lat, lon) + if forecast is None: + self.dbus_service['/Status'] = 3 + self.dbus_service['/ErrorMessage'] = 'API returned no data' + return + + forecast_json = json.dumps(forecast) + self.dbus_service['/Forecast/Json'] = forecast_json + self.dbus_service['/Forecast/Latitude'] = lat + self.dbus_service['/Forecast/Longitude'] = lon + self.dbus_service['/LastUpdate'] = int(time.time()) + self.dbus_service['/ErrorMessage'] = '' + self.dbus_service['/Status'] = 2 + + ts_count = len(forecast.get('ts', [])) + self.logger.info( + f"Forecast updated: {ts_count} timestamps, " + f"{len(forecast_json)} bytes") + + except Exception as e: + self.logger.exception(f"Fetch error: {e}") + self.dbus_service['/Status'] = 3 + self.dbus_service['/ErrorMessage'] = str(e)[:200] + finally: + self.last_fetch_time = time.time() + self._fetch_in_progress = False + self._update_next_time() + + def _update_next_time(self): + moving = self.dbus_service['/IsMoving'] + interval = (REFRESH_INTERVAL_MOVING if moving + else REFRESH_INTERVAL_STATIONARY) + next_time = self.last_fetch_time + interval + self.dbus_service['/NextUpdate'] = int(next_time) + + def _main_loop(self): + try: + if not self.enabled: + self.dbus_service['/Status'] = 0 + return True + + now = time.time() + + if now - self.last_gps_check >= GPS_SAMPLE_INTERVAL: + pos = self.gps.get_position() + if pos: + self.current_lat, self.current_lon = pos + if now - self.last_gps_history_sample >= GPS_HISTORY_SAMPLE_INTERVAL: + self.movement.add_position( + self.current_lat, self.current_lon, now) + self.last_gps_history_sample = now + is_moving = self.movement.is_moving( + self.current_lat, self.current_lon) + self.dbus_service['/IsMoving'] = 1 if is_moving else 0 + self.last_gps_check = now + + if (self.current_lat is not None + and self.current_lon is not None + and self.api_key + and not self._fetch_in_progress): + is_moving = self.dbus_service['/IsMoving'] + interval = (REFRESH_INTERVAL_MOVING if is_moving + else REFRESH_INTERVAL_STATIONARY) + if self.last_fetch_time == 0 or now - self.last_fetch_time >= interval: + self._fetch_in_progress = True + thread = threading.Thread( + target=self._do_fetch, + args=(self.current_lat, self.current_lon), + daemon=True) + thread.start() + + if not self.api_key: + self.dbus_service['/Status'] = 0 + self.dbus_service['/ErrorMessage'] = 'API key not configured' + elif self.current_lat is None: + if self.dbus_service['/Status'] != 2: + self.dbus_service['/Status'] = 0 + self.dbus_service['/ErrorMessage'] = 'Waiting for GPS fix' + + except dbus.exceptions.DBusException as e: + self.logger.warning(f"D-Bus error: {e}") + except Exception as e: + self.logger.exception(f"Unexpected error: {e}") + + return True + + +def main(): + DBusGMainLoop(set_as_default=True) + + print("=" * 60) + print(f"Meteoblue Forecast v{VERSION}") + print("=" * 60) + + mainloop = None + + def signal_handler(signum, frame): + try: + sig_name = signal.Signals(signum).name + except ValueError: + sig_name = str(signum) + logging.info(f"Received {sig_name}, shutting down...") + if mainloop is not None: + mainloop.quit() + + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + + try: + controller = MeteoblueForecastController() + mainloop = GLib.MainLoop() + mainloop.run() + except KeyboardInterrupt: + print("\nShutdown requested") + except Exception as e: + logging.error(f"Fatal error: {e}", exc_info=True) + sys.exit(1) + finally: + logging.info("Service stopped") + + +if __name__ == '__main__': + main() diff --git a/dbus-meteoblue-forecast/service/log/run b/dbus-meteoblue-forecast/service/log/run new file mode 100644 index 0000000..de3c565 --- /dev/null +++ b/dbus-meteoblue-forecast/service/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec multilog t s99999 n8 /var/log/dbus-meteoblue-forecast diff --git a/dbus-meteoblue-forecast/service/run b/dbus-meteoblue-forecast/service/run new file mode 100644 index 0000000..83b13e3 --- /dev/null +++ b/dbus-meteoblue-forecast/service/run @@ -0,0 +1,5 @@ +#!/bin/sh +exec 2>&1 +cd /data/dbus-meteoblue-forecast +export PYTHONPATH="/data/dbus-meteoblue-forecast/ext/velib_python:$PYTHONPATH" +exec python3 /data/dbus-meteoblue-forecast/meteoblue_forecast.py diff --git a/dbus-meteoblue-forecast/uninstall.sh b/dbus-meteoblue-forecast/uninstall.sh new file mode 100644 index 0000000..26c584c --- /dev/null +++ b/dbus-meteoblue-forecast/uninstall.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Uninstall Meteoblue Forecast for Venus OS + +INSTALL_DIR="/data/dbus-meteoblue-forecast" +SERVICE_LINK="dbus-meteoblue-forecast" + +# Find service directory +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "Uninstalling Meteoblue Forecast..." + +# Stop and remove service +if [ -L "$SERVICE_DIR/$SERVICE_LINK" ] || [ -e "$SERVICE_DIR/$SERVICE_LINK" ]; then + echo "Stopping and removing service..." + svc -d "$SERVICE_DIR/$SERVICE_LINK" 2>/dev/null || true + rm -f "$SERVICE_DIR/$SERVICE_LINK" + rm -rf "$SERVICE_DIR/$SERVICE_LINK" +fi + +echo "Service removed. Config and data in $INSTALL_DIR are preserved." +echo "To remove everything: rm -rf $INSTALL_DIR /var/log/dbus-meteoblue-forecast" diff --git a/dbus-no-foreign-land/.gitignore b/dbus-no-foreign-land/.gitignore new file mode 100644 index 0000000..b88ffb9 --- /dev/null +++ b/dbus-no-foreign-land/.gitignore @@ -0,0 +1,55 @@ +# Logs +logs +*.log +npm-debug.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# node-waf configuration +.lock-wscript + +dist + +# Compiled binary addons (http://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules +jspm_packages + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +.DS_Store +*.db + +#npm packaged files +*.tgz \ No newline at end of file diff --git a/dbus-no-foreign-land/.prettierrc b/dbus-no-foreign-land/.prettierrc new file mode 100644 index 0000000..3de448e --- /dev/null +++ b/dbus-no-foreign-land/.prettierrc @@ -0,0 +1,7 @@ +{ + "semi": true, + "trailingComma": "es5", + "singleQuote": true, + "printWidth": 100, + "tabWidth": 2 +} diff --git a/dbus-no-foreign-land/DbusSettingsList b/dbus-no-foreign-land/DbusSettingsList new file mode 100644 index 0000000..17b0ccf --- /dev/null +++ b/dbus-no-foreign-land/DbusSettingsList @@ -0,0 +1,3 @@ +{"path":"/Settings/NflTracking/BoatApiKey", "default":""} +{"path":"/Settings/NflTracking/BoatName", "default":""} +{"path":"/Settings/NflTracking/Enabled", "default":1, "min":0, "max":1} diff --git a/dbus-no-foreign-land/NFL_API_REFERENCE.md b/dbus-no-foreign-land/NFL_API_REFERENCE.md new file mode 100644 index 0000000..a952e8e --- /dev/null +++ b/dbus-no-foreign-land/NFL_API_REFERENCE.md @@ -0,0 +1,249 @@ +# NFL (No Foreign Land) API Reference for Location & Tracking + +This document describes how the No Foreign Land (NFL) API works for submitting boat location and tracking data. Use it to build your own simpler module that does the same thing. + +--- + +## Overview + +The NFL API accepts **track data** (sequences of GPS points) via a single HTTP POST endpoint. Each submission includes: + +1. A **boat API key** (user-specific, from noforeignland.com) +2. A **timestamp** (milliseconds since epoch) +3. A **track** (array of `[timestamp_ms, lat, lon]` points) + +--- + +## API Endpoint + +| Property | Value | +|----------|-------| +| **URL** | `https://www.noforeignland.com/home/api/v1/boat/tracking/track` | +| **Method** | `POST` | +| **Content-Type** | `application/x-www-form-urlencoded` | + +--- + +## Authentication + +### 1. Plugin API Key (hardcoded) + +The Signal K plugin uses a fixed plugin API key sent in the header: + +``` +X-NFL-API-Key: 0ede6cb6-5213-45f5-8ab4-b4836b236f97 +``` + +This identifies the plugin/client to the NFL backend. Your own module may need to use the same key or obtain a different one from NFL. + +### 2. Boat API Key (user-specific) + +Each boat/user has a **Boat API Key** from noforeignland.com: + +- **Where to get it:** Account > Settings > Boat tracking > API Key (on the website only, not in the app) +- **How it’s sent:** As a form field `boatApiKey` in the POST body + +--- + +## Request Format + +### POST Body (form-urlencoded) + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `timestamp` | string | Yes | Unix timestamp in **milliseconds** (epoch of the last track point) | +| `track` | string | Yes | JSON string of track points (see below) | +| `boatApiKey` | string | Yes | User’s boat API key from noforeignland.com | + +### Track format + +`track` is a **JSON string** (so it must be serialized before being sent as form data). Example: + +```json +[ + [1234567890000, 52.1234, 4.5678], + [1234567896000, 52.1240, 4.5680], + [1234567902000, 52.1245, 4.5685] +] +``` + +Each point is a **3-element array**: + +| Index | Type | Description | +|-------|------|--------------| +| 0 | number | Unix timestamp in **milliseconds** | +| 1 | number | Latitude (WGS84, -90 to 90) | +| 2 | number | Longitude (WGS84, -180 to 180) | + +Points should be ordered by time (oldest first). + +--- + +## Response Format + +```json +{ + "status": "ok", + "message": "optional message" +} +``` + +Or on error: + +```json +{ + "status": "error", + "message": "Error description" +} +``` + +### HTTP Status Codes + +- **200**: Request accepted. Check `status` in the JSON body for success/failure. +- **4xx**: Client error (e.g. invalid API key, bad request). Do not retry. +- **5xx**: Server error. Retry with backoff. + +--- + +## Minimal Example (cURL) + +```bash +curl -X POST "https://www.noforeignland.com/home/api/v1/boat/tracking/track" \ + -H "X-NFL-API-Key: 0ede6cb6-5213-45f5-8ab4-b4836b236f97" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "timestamp=1234567902000" \ + -d "track=[[1234567890000,52.1234,4.5678],[1234567896000,52.1240,4.5680],[1234567902000,52.1245,4.5685]]" \ + -d "boatApiKey=YOUR_BOAT_API_KEY" +``` + +--- + +## Minimal Example (JavaScript/Node.js) + +```javascript +const track = [ + [Date.now() - 60000, 52.1234, 4.5678], // 1 min ago + [Date.now() - 30000, 52.1240, 4.5680], // 30 sec ago + [Date.now(), 52.1245, 4.5685] // now +]; + +const lastTimestamp = track[track.length - 1][0]; + +const params = new URLSearchParams(); +params.append('timestamp', String(lastTimestamp)); +params.append('track', JSON.stringify(track)); +params.append('boatApiKey', 'YOUR_BOAT_API_KEY'); + +const response = await fetch('https://www.noforeignland.com/home/api/v1/boat/tracking/track', { + method: 'POST', + headers: { + 'X-NFL-API-Key': '0ede6cb6-5213-45f5-8ab4-b4836b236f97', + }, + body: params, +}); + +const result = await response.json(); +if (response.ok && result.status === 'ok') { + console.log('Track sent successfully'); +} else { + console.error('Error:', result.message || response.statusText); +} +``` + +--- + +## Minimal Example (Python) + +```python +import requests +import json +import time + +track = [ + [int(time.time() * 1000) - 60000, 52.1234, 4.5678], # 1 min ago + [int(time.time() * 1000) - 30000, 52.1240, 4.5680], # 30 sec ago + [int(time.time() * 1000), 52.1245, 4.5685] # now +] + +last_timestamp = track[-1][0] + +response = requests.post( + 'https://www.noforeignland.com/home/api/v1/boat/tracking/track', + headers={'X-NFL-API-Key': '0ede6cb6-5213-45f5-8ab4-b4836b236f97'}, + data={ + 'timestamp': last_timestamp, + 'track': json.dumps(track), + 'boatApiKey': 'YOUR_BOAT_API_KEY' + } +) + +result = response.json() +if response.ok and result.get('status') == 'ok': + print('Track sent successfully') +else: + print('Error:', result.get('message', response.text)) +``` + +--- + +## Data Validation (from the Signal K plugin) + +### Position validation + +- Latitude: -90 to 90 +- Longitude: -180 to 180 +- Reject non-numeric, NaN, Infinity +- Reject positions near (0, 0) (likely GPS init values) + +### Velocity filter (optional) + +To catch GPS outliers, the plugin rejects points that imply movement faster than a threshold (default 50 m/s ≈ 97 knots): + +``` +velocity = distance_meters / time_delta_seconds +``` + +If `velocity > maxVelocity`, the point is dropped. + +--- + +## 24h Keepalive + +The plugin does not use a separate keepalive endpoint. Instead: + +- If a boat has not moved for 24 hours, it still saves a point (using the last known position). +- That point is included in the next track upload. +- The effect is that boats stay “active” on NFL even when stationary. + +--- + +## Retry Logic (from the plugin) + +- Retry up to 3 times on network errors or 5xx responses. +- Do not retry on 4xx client errors. +- Increase timeout per attempt (e.g. 30s, 60s, 90s). +- Wait 2–4–6 seconds between retries. + +--- + +## Summary: Minimal Module Checklist + +| Requirement | Details | +|-------------|---------| +| Endpoint | `POST https://www.noforeignland.com/home/api/v1/boat/tracking/track` | +| Header | `X-NFL-API-Key: 0ede6cb6-5213-45f5-8ab4-b4836b236f97` | +| Body | `application/x-www-form-urlencoded` with `timestamp`, `track`, `boatApiKey` | +| Track format | JSON array of `[timestamp_ms, lat, lon]` | +| Success | `response.status === 200` and `body.status === 'ok'` | + +--- + +## Source Files Reference + +| File | Purpose | +|------|---------| +| `src/types/api.ts` | API URL, plugin key, response types | +| `src/lib/TrackSender.ts` | HTTP request, retry logic, track parsing | +| `src/lib/TrackLogger.ts` | Position collection, filtering, JSONL storage | +| `src/types/position.ts` | Internal position format | +| `src/utils/validation.ts` | Position and velocity checks | diff --git a/dbus-no-foreign-land/README.md b/dbus-no-foreign-land/README.md new file mode 100644 index 0000000..b580f92 --- /dev/null +++ b/dbus-no-foreign-land/README.md @@ -0,0 +1,115 @@ +# NFL Tracking for Venus OS + +A Venus OS addon that sends boat position and track data to [noforeignland.com](https://www.noforeignland.com) without requiring Signal K. + +## Features + +- Reads GPS position from Venus OS D-Bus (uses built-in dbus-python, no extra downloads) +- Sends track data to the NFL API +- Configurable via Settings menu (Settings → GPS → NFL Tracking) +- 24h keepalive when boat is stationary +- Runs as a daemon with automatic restart + +## Requirements + +- Venus OS device (Cerbo GX, Venus GX, etc.) with root access +- Python 3 with dbus-python (built into Venus OS) +- GPS connected to Venus OS (USB/serial GPS or router GPS) +- noforeignland.com account and Boat API Key (Account > Settings > Boat tracking > API Key) +- Internet connection for API uploads + +## Installation + +### From built package + +```bash +# On your computer - build the package: +cd dbus-no-foreign-land +./build-package.sh --version 1.0.0 + +# Copy to Venus OS device: +scp dbus-no-foreign-land-1.0.0.tar.gz root@:/data/ + +# SSH to device and install: +ssh root@ +cd /data && tar -xzf dbus-no-foreign-land-1.0.0.tar.gz +mv dbus-no-foreign-land /data/dbus-no-foreign-land +bash /data/dbus-no-foreign-land/install.sh +bash /data/dbus-no-foreign-land/install_gui.sh # Add to Settings menu +# Configure via Settings → GPS → NFL Tracking (API key, boat name, intervals) +``` + +### Manual install (from source) + +1. Copy the project folder to `/data/dbus-no-foreign-land` on your Venus OS device +2. Run `bash /data/dbus-no-foreign-land/install.sh` +3. Run `bash /data/dbus-no-foreign-land/install_gui.sh` to add NFL Tracking to Settings → GPS +4. Configure via **Settings → GPS → NFL Tracking** (API key, boat name, intervals) + +## Configuration + +All settings are configured via the Venus GUI: **Settings → GPS → NFL Tracking** + +- **Boat API Key** – Get from noforeignland.com: Account > Settings > Boat tracking > API Key +- **Boat name** – Optional display name +- **Enable tracking** – Master on/off switch +- **Track interval** – Minimum seconds between points (10–3600s) +- **Min distance** – Meters moved before logging new point (1–10000m) +- **Send interval** – Minutes between API uploads (1–1440min) +- **24h keepalive** – Send a point every 24h when stationary + +Settings are stored in Venus localsettings (D-Bus) and persist across reboots. + +**Alternative:** Configure via MQTT: `N//nfltracking/0/Settings/BoatApiKey`, etc. + +## GUI Settings + +After `install.sh`, run `install_gui.sh` to add NFL Tracking to Settings → GPS: + +```bash +bash /data/dbus-no-foreign-land/install_gui.sh +``` + +If the GUI goes blank, run `install_gui.sh --remove` to restore. + +## GPS Data Source + +The addon reads GPS from Venus OS D-Bus: + +1. Reads `com.victronenergy.system` → `/GpsService` to get the best GPS service name (e.g. `com.victronenergy.gps.ve_ttyUSB0`) +2. Reads position from that GPS service: + - Prefers `/Latitude` and `/Longitude` (numeric) + - Fallback: `/Position` (string `"lat,lon"`) +3. Requires `/Fix` = 1 (GPS fix) before using position + +If GPS data is not received: +- Ensure GPS is connected and has a fix (Device List > Boat > GPS) +- Run `dbus -y` on the device and inspect `com.victronenergy.gps.*` to verify paths +- Some GPS sources (e.g. router NMEA) may use different path names + +## Debugging + +```bash +# View logs +tail -F /var/log/dbus-no-foreign-land/current | tai64nlocal + +# Restart service +/data/dbus-no-foreign-land/restart.sh + +# Check status +svstat /service/dbus-no-foreign-land +``` + +## Troubleshooting + +**"MemoryError" or "Cannot allocate memory"** – Python may fail to start on very low-RAM devices. Check free memory: `free -m` + +## Uninstall + +```bash +bash /data/dbus-no-foreign-land/uninstall.sh +``` + +## License + +MIT diff --git a/dbus-no-foreign-land/build-package.sh b/dbus-no-foreign-land/build-package.sh new file mode 100755 index 0000000..674154f --- /dev/null +++ b/dbus-no-foreign-land/build-package.sh @@ -0,0 +1,218 @@ +#!/bin/bash +# +# Build script for NFL Tracking Venus OS package +# +# Creates a tar.gz package that can be: +# 1. Copied to a Venus OS device (Cerbo GX, Venus GX, etc.) +# 2. Untarred to /data/ +# 3. Installed by running install.sh +# +# Usage: +# ./build-package.sh # Creates package with default name +# ./build-package.sh --version 1.0.0 # Creates package with version in name +# ./build-package.sh --output /path/ # Specify output directory +# +# Output: dbus-no-foreign-land-.tar.gz +# +# Installation on Venus OS: +# scp dbus-no-foreign-land-*.tar.gz root@:/data/ +# ssh root@ +# cd /data && tar -xzf dbus-no-foreign-land-*.tar.gz +# mv dbus-no-foreign-land /data/dbus-no-foreign-land +# bash /data/dbus-no-foreign-land/install.sh +# bash /data/dbus-no-foreign-land/install_gui.sh +# Configure via Settings > GPS > NFL Tracking +# + +set -e + +# Script directory (where the source files are) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Default values +VERSION="1.0.0" +OUTPUT_DIR="$SCRIPT_DIR" +PACKAGE_NAME="dbus-no-foreign-land" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --version|-v) + VERSION="$2" + shift 2 + ;; + --output|-o) + OUTPUT_DIR="$2" + shift 2 + ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " -v, --version VERSION Set package version (default: 1.0.0)" + echo " -o, --output PATH Output directory (default: script directory)" + echo " -h, --help Show this help message" + echo "" + echo "Example:" + echo " $0 --version 1.2.0 --output ./dist/" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Build timestamp +BUILD_DATE=$(date -u +"%Y-%m-%d %H:%M:%S UTC") +BUILD_TIMESTAMP=$(date +%Y%m%d%H%M%S) + +# Temporary build directory +BUILD_DIR=$(mktemp -d) +PACKAGE_DIR="$BUILD_DIR/$PACKAGE_NAME" + +echo "==================================================" +echo "Building $PACKAGE_NAME package" +echo "==================================================" +echo "Version: $VERSION" +echo "Build date: $BUILD_DATE" +echo "Source: $SCRIPT_DIR" +echo "Output: $OUTPUT_DIR" +echo "" + +# Create package directory structure (aligned with dbus-generator-ramp) +echo "1. Creating package structure..." +mkdir -p "$PACKAGE_DIR" +mkdir -p "$PACKAGE_DIR/service/log" +mkdir -p "$PACKAGE_DIR/qml" + +# On macOS, prevent cp from copying extended attributes (avoids ._* and com.apple.provenance) +[ "$(uname)" = "Darwin" ] && export COPYFILE_DISABLE=1 + +# Copy application files +echo "2. Copying application files..." +cp "$SCRIPT_DIR/nfl_tracking.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/config.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/config.sample.ini" "$PACKAGE_DIR/" + +# Copy service files +echo "3. Copying service files..." +cp "$SCRIPT_DIR/service/run" "$PACKAGE_DIR/service/" +cp "$SCRIPT_DIR/service/log/run" "$PACKAGE_DIR/service/log/" + +# Copy QML files +echo "4. Copying GUI files..." +if [ -f "$SCRIPT_DIR/qml/PageSettingsNflTracking.qml" ]; then + cp "$SCRIPT_DIR/qml/PageSettingsNflTracking.qml" "$PACKAGE_DIR/qml/" +fi + +# Copy installation scripts +echo "5. Copying installation scripts..." +cp "$SCRIPT_DIR/install.sh" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/uninstall.sh" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/install_gui.sh" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/restart.sh" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/download.sh" "$PACKAGE_DIR/" + +# Copy DbusSettingsList (for SetupHelper integration) +if [ -f "$SCRIPT_DIR/DbusSettingsList" ]; then + cp "$SCRIPT_DIR/DbusSettingsList" "$PACKAGE_DIR/" +fi + +# Copy documentation +echo "6. Copying documentation..." +cp "$SCRIPT_DIR/README.md" "$PACKAGE_DIR/" + +# Create version file with build info +echo "7. Creating version info..." +cat > "$PACKAGE_DIR/VERSION" << EOF +Package: $PACKAGE_NAME +Version: $VERSION +Build Date: $BUILD_DATE +Build Timestamp: $BUILD_TIMESTAMP + +Installation: + 1. Copy to Venus OS: scp $PACKAGE_NAME-$VERSION.tar.gz root@:/data/ + 2. SSH to device: ssh root@ + 3. Extract: cd /data && tar -xzf $PACKAGE_NAME-$VERSION.tar.gz + 4. Move: mv $PACKAGE_NAME /data/dbus-no-foreign-land + 5. Install: bash /data/dbus-no-foreign-land/install.sh + 6. GUI: bash /data/dbus-no-foreign-land/install_gui.sh + 7. Configure via Settings > GPS > NFL Tracking + +For more information, see README.md +EOF + +# Set executable permissions +echo "8. Setting permissions..." +chmod +x "$PACKAGE_DIR/nfl_tracking.py" +chmod +x "$PACKAGE_DIR/install.sh" +chmod +x "$PACKAGE_DIR/uninstall.sh" +chmod +x "$PACKAGE_DIR/install_gui.sh" +chmod +x "$PACKAGE_DIR/restart.sh" +chmod +x "$PACKAGE_DIR/download.sh" +chmod +x "$PACKAGE_DIR/service/run" +chmod +x "$PACKAGE_DIR/service/log/run" + +# Create output directory if needed +mkdir -p "$OUTPUT_DIR" + +# Create the tar.gz package +TARBALL_NAME="$PACKAGE_NAME-$VERSION.tar.gz" +OUTPUT_DIR_ABS="$(cd "$OUTPUT_DIR" && pwd)" +TARBALL_PATH="$OUTPUT_DIR_ABS/$TARBALL_NAME" + +echo "9. Creating package archive..." +cd "$BUILD_DIR" +if [ "$(uname)" = "Darwin" ]; then + # Strip macOS extended attributes (com.apple.provenance, etc.) before tarring + # to avoid "Ignoring unknown extended header keyword" warnings on Linux + if command -v xattr >/dev/null 2>&1; then + xattr -cr "$PACKAGE_NAME" + fi +fi +# Use POSIX ustar format - does not store extended attributes, avoids Linux extraction warnings +tar --format=ustar -czf "$TARBALL_PATH" "$PACKAGE_NAME" + +# Calculate checksum (sha256sum on Linux, shasum on macOS) +if command -v sha256sum >/dev/null 2>&1; then + CHECKSUM=$(sha256sum "$TARBALL_PATH" | cut -d' ' -f1) +else + CHECKSUM=$(shasum -a 256 "$TARBALL_PATH" | cut -d' ' -f1) +fi + +# Create checksum file +echo "$CHECKSUM $TARBALL_NAME" > "$OUTPUT_DIR_ABS/$TARBALL_NAME.sha256" + +# Clean up +echo "10. Cleaning up..." +rm -rf "$BUILD_DIR" + +# Get file size +if [ "$(uname)" = "Darwin" ]; then + FILE_SIZE=$(du -h "$TARBALL_PATH" | cut -f1) +else + FILE_SIZE=$(du -h "$TARBALL_PATH" | cut -f1) +fi + +echo "" +echo "==================================================" +echo "Build complete!" +echo "==================================================" +echo "" +echo "Package: $TARBALL_PATH" +echo "Size: $FILE_SIZE" +echo "SHA256: $CHECKSUM" +echo "" +echo "Installation on Venus OS:" +echo " scp $TARBALL_PATH root@:/data/" +echo " ssh root@" +echo " cd /data" +echo " tar -xzf $TARBALL_NAME" +echo " mv $PACKAGE_NAME /data/dbus-no-foreign-land" +echo " bash /data/dbus-no-foreign-land/install.sh" +echo " bash /data/dbus-no-foreign-land/install_gui.sh" +echo " Configure via Settings > GPS > NFL Tracking" +echo "" diff --git a/dbus-no-foreign-land/config.py b/dbus-no-foreign-land/config.py new file mode 100644 index 0000000..ac5790a --- /dev/null +++ b/dbus-no-foreign-land/config.py @@ -0,0 +1,68 @@ +""" +Configuration for NFL (No Foreign Land) Tracking + +All tunable parameters in one place for easy adjustment. +""" + +# ============================================================================= +# D-BUS SERVICE CONFIGURATION +# ============================================================================= + +# D-Bus service name for our addon (visible as N//nfltracking/0/...) +SERVICE_NAME = 'com.victronenergy.nfltracking' + +# ============================================================================= +# NFL API CONFIGURATION +# ============================================================================= + +NFL_API_URL = "https://www.noforeignland.com/home/api/v1/boat/tracking/track" +NFL_PLUGIN_API_KEY = "0ede6cb6-5213-45f5-8ab4-b4836b236f97" + +# ============================================================================= +# TRACKING CONFIGURATION +# ============================================================================= + +TRACKING_CONFIG = { + # Minimum interval between track points (seconds) + 'track_interval': 60, + + # Minimum distance moved before logging new point (meters) + 'min_distance': 50, + + # How often to send track to API (minutes) + 'send_interval': 15, + + # 24h keepalive: log a point every 24h even when stationary + 'ping_24h': True, + + # Poll interval for main loop (seconds) - how often we check GPS + 'poll_interval': 30, + + # Max track points in memory/file (prevents OOM on Cerbo if sends fail) + 'max_track_points': 5000, +} + +# ============================================================================= +# PATH CONFIGURATION +# ============================================================================= + +# Config and data directory (on Venus OS) +DATA_DIR = '/data/dbus-no-foreign-land' +CONFIG_DIR = DATA_DIR +TRACK_FILE = f'{CONFIG_DIR}/pending.jsonl' +CONFIG_FILE = f'{CONFIG_DIR}/config.ini' + +# ============================================================================= +# LOGGING CONFIGURATION +# ============================================================================= + +LOGGING_CONFIG = { + # Log level: DEBUG, INFO, WARNING, ERROR + 'level': 'INFO', + + # Log to console (stdout) + 'console': True, + + # Include timestamps in console output (multilog adds its own on Venus) + 'include_timestamp': False, +} diff --git a/dbus-no-foreign-land/config.sample.ini b/dbus-no-foreign-land/config.sample.ini new file mode 100644 index 0000000..8b7a453 --- /dev/null +++ b/dbus-no-foreign-land/config.sample.ini @@ -0,0 +1,25 @@ +[DEFAULT] +# DEPRECATED: Settings are now configured via Venus GUI (Settings > GPS > NFL Tracking). +# This file is kept for reference only. Do not edit - it is not read by the service. + +# Your Boat API Key from noforeignland.com +# Get it from: Account > Settings > Boat tracking > API Key (website only, not in app) +boat_api_key = + +# Optional: Boat name for display +boat_name = + +# Track logging: minimum interval between points (seconds) +track_interval = 60 + +# Track logging: minimum distance moved before logging (meters) +min_distance = 50 + +# How often to send track to API (minutes) +send_interval = 15 + +# 24h keepalive: send a point every 24h even when stationary +ping_24h = true + +# Logging level: DEBUG, INFO, WARNING, ERROR +logging = INFO diff --git a/dbus-no-foreign-land/download.sh b/dbus-no-foreign-land/download.sh new file mode 100644 index 0000000..2d61742 --- /dev/null +++ b/dbus-no-foreign-land/download.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Download and install NFL Tracking for Venus OS +# Run: wget -O /tmp/install.sh https://raw.githubusercontent.com/.../download.sh && bash /tmp/install.sh + +set -e +REPO_URL="${REPO_URL:-https://github.com/noforeignland/nfl-signalk}" +BRANCH="${BRANCH:-main}" +INSTALL_DIR="/data/dbus-no-foreign-land" +TMP_DIR="/tmp/nfl-tracking-install" + +echo "Downloading NFL Tracking..." + +mkdir -p "${TMP_DIR}" +cd "${TMP_DIR}" + +# Fetch the venus-nfl-tracking folder from the repo +if command -v wget >/dev/null 2>&1; then + wget -q -O venus-nfl-tracking.tar.gz "${REPO_URL}/archive/refs/heads/${BRANCH}.tar.gz" || { + echo "Failed to download. Check REPO_URL and network." + exit 1 + } + tar xzf venus-nfl-tracking.tar.gz + SRC="${TMP_DIR}/nfl-signalk-${BRANCH}/venus-nfl-tracking" +elif command -v curl >/dev/null 2>&1; then + curl -sL -o venus-nfl-tracking.tar.gz "${REPO_URL}/archive/refs/heads/${BRANCH}.tar.gz" || { + echo "Failed to download. Check REPO_URL and network." + exit 1 + } + tar xzf venus-nfl-tracking.tar.gz + SRC="${TMP_DIR}/nfl-signalk-${BRANCH}/venus-nfl-tracking" +else + echo "Need wget or curl" + exit 1 +fi + +if [ ! -d "${SRC}" ]; then + echo "Source not found at ${SRC}" + exit 1 +fi + +mkdir -p "${INSTALL_DIR}" +cp -r "${SRC}"/* "${INSTALL_DIR}/" +rm -rf "${TMP_DIR}" + +echo "Running install.sh..." +bash "${INSTALL_DIR}/install.sh" diff --git a/dbus-no-foreign-land/install.sh b/dbus-no-foreign-land/install.sh new file mode 100644 index 0000000..c43e741 --- /dev/null +++ b/dbus-no-foreign-land/install.sh @@ -0,0 +1,198 @@ +#!/bin/bash +# +# Installation script for NFL (No Foreign Land) Tracking +# +# Run this on the Venus OS device after copying files to /data/dbus-no-foreign-land/ +# +# Usage: +# chmod +x install.sh +# ./install.sh +# + +set -e + +INSTALL_DIR="/data/dbus-no-foreign-land" + +# Find velib_python - check standard location first, then search existing services +VELIB_DIR="" +if [ -d "/opt/victronenergy/velib_python" ]; then + VELIB_DIR="/opt/victronenergy/velib_python" +else + for candidate in \ + "/opt/victronenergy/dbus-systemcalc-py/ext/velib_python" \ + "/opt/victronenergy/dbus-generator/ext/velib_python" \ + "/opt/victronenergy/dbus-mqtt/ext/velib_python" \ + "/opt/victronenergy/dbus-digitalinputs/ext/velib_python" \ + "/opt/victronenergy/vrmlogger/ext/velib_python" + do + if [ -d "$candidate" ] && [ -f "$candidate/vedbus.py" ]; then + VELIB_DIR="$candidate" + break + fi + done +fi + +if [ -z "$VELIB_DIR" ]; then + VEDBUS_PATH=$(find /opt/victronenergy -name "vedbus.py" -path "*/velib_python/*" 2>/dev/null | head -1) + if [ -n "$VEDBUS_PATH" ]; then + VELIB_DIR=$(dirname "$VEDBUS_PATH") + fi +fi + +# Determine the correct service directory +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +elif [ -L "/service" ]; then + SERVICE_DIR=$(readlink -f /service) +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "==================================================" +echo "NFL (No Foreign Land) Tracking - Installation" +echo "==================================================" + +# Check if running on Venus OS +if [ ! -d "$SERVICE_DIR" ]; then + echo "ERROR: This doesn't appear to be a Venus OS device." + echo " Service directory not found." + echo " Checked: /service, /opt/victronenergy/service" + exit 1 +fi + +echo "Detected service directory: $SERVICE_DIR" + +# Check if required files exist +if [ ! -f "$INSTALL_DIR/nfl_tracking.py" ]; then + echo "ERROR: Installation files not found in $INSTALL_DIR" + echo " Please copy all files to $INSTALL_DIR first." + exit 1 +fi +if [ ! -f "$INSTALL_DIR/service/run" ]; then + echo "ERROR: service/run not found. The package is incomplete." + echo " Rebuild with build-package.sh or ensure service/ is included." + echo " Expected: $INSTALL_DIR/service/run" + exit 1 +fi + +echo "1. Making scripts executable..." +chmod +x "$INSTALL_DIR/service/run" +chmod +x "$INSTALL_DIR/service/log/run" +chmod +x "$INSTALL_DIR/nfl_tracking.py" +if [ -f "$INSTALL_DIR/install_gui.sh" ]; then + chmod +x "$INSTALL_DIR/install_gui.sh" +fi + +echo "2. Creating velib_python symlink..." +if [ -z "$VELIB_DIR" ]; then + echo "ERROR: Could not find velib_python on this system." + echo " Searched /opt/victronenergy for vedbus.py" + exit 1 +fi +echo " Found velib_python at: $VELIB_DIR" +mkdir -p "$INSTALL_DIR/ext" +if [ -L "$INSTALL_DIR/ext/velib_python" ]; then + CURRENT_TARGET=$(readlink "$INSTALL_DIR/ext/velib_python") + if [ "$CURRENT_TARGET" != "$VELIB_DIR" ]; then + echo " Updating symlink (was: $CURRENT_TARGET)" + rm "$INSTALL_DIR/ext/velib_python" + fi +fi +if [ ! -L "$INSTALL_DIR/ext/velib_python" ]; then + ln -s "$VELIB_DIR" "$INSTALL_DIR/ext/velib_python" + echo " Symlink created: $INSTALL_DIR/ext/velib_python -> $VELIB_DIR" +else + echo " Symlink already exists" +fi + +echo "3. Creating main service symlink..." +if [ -L "$SERVICE_DIR/dbus-no-foreign-land" ]; then + echo " Service link already exists, removing old link..." + rm "$SERVICE_DIR/dbus-no-foreign-land" +fi +if [ -e "$SERVICE_DIR/dbus-no-foreign-land" ]; then + echo " Removing existing service directory..." + rm -rf "$SERVICE_DIR/dbus-no-foreign-land" +fi +ln -s "$INSTALL_DIR/service" "$SERVICE_DIR/dbus-no-foreign-land" + +# Verify symlink was created +if [ -L "$SERVICE_DIR/dbus-no-foreign-land" ]; then + echo " Symlink created: $SERVICE_DIR/dbus-no-foreign-land -> $INSTALL_DIR/service" +else + echo "ERROR: Failed to create service symlink" + exit 1 +fi + +echo "4. Creating data and log directories..." +mkdir -p "$INSTALL_DIR" +mkdir -p /var/log/dbus-no-foreign-land + +echo "5. Setting up rc.local for persistence..." +RC_LOCAL="/data/rc.local" +if [ ! -f "$RC_LOCAL" ]; then + echo "#!/bin/bash" > "$RC_LOCAL" + chmod +x "$RC_LOCAL" +fi + +if ! grep -q "dbus-no-foreign-land" "$RC_LOCAL"; then + echo "" >> "$RC_LOCAL" + echo "# NFL (No Foreign Land) Tracking" >> "$RC_LOCAL" + echo "if [ ! -L $SERVICE_DIR/dbus-no-foreign-land ]; then" >> "$RC_LOCAL" + echo " ln -s /data/dbus-no-foreign-land/service $SERVICE_DIR/dbus-no-foreign-land" >> "$RC_LOCAL" + echo "fi" >> "$RC_LOCAL" + echo " Added to rc.local for persistence across firmware updates" +else + echo " Already in rc.local" +fi + +echo "6. Activating service..." +sleep 2 +if command -v svstat >/dev/null 2>&1; then + if svstat "$SERVICE_DIR/dbus-no-foreign-land" 2>/dev/null | grep -q "up"; then + echo " Service is running" + else + echo " Waiting for service to start..." + sleep 3 + fi +fi + +echo "" +echo "==================================================" +echo "Installation complete!" +echo "==================================================" +echo "" +echo "Service directory: $SERVICE_DIR" +echo "" + +if command -v svstat >/dev/null 2>&1; then + echo "Current status:" + svstat "$SERVICE_DIR/dbus-no-foreign-land" 2>/dev/null || echo " Service not yet detected by svscan" + echo "" +fi + +echo "IMPORTANT: Configure via Settings menu:" +echo " 1. Run: ./install_gui.sh (adds NFL Tracking to Settings > GPS)" +echo " 2. Open Settings > GPS > NFL Tracking and enter your Boat API Key" +echo " 3. Or use MQTT: N//nfltracking/0/Settings/BoatApiKey" +echo "" +echo "To check status:" +echo " svstat $SERVICE_DIR/dbus-no-foreign-land" +echo "" +echo "To view logs:" +echo " tail -F /var/log/dbus-no-foreign-land/current | tai64nlocal" +echo "" +echo "MQTT Paths:" +echo " N//nfltracking/0/Status" +echo " N//nfltracking/0/TrackPoints" +echo " N//nfltracking/0/Settings/BoatApiKey" +echo "" +echo "Install GUI integration (required for configuration):" +echo " ./install_gui.sh" +echo "" +echo "Troubleshooting:" +echo " ls -la $SERVICE_DIR/dbus-no-foreign-land" +echo " cat /var/log/dbus-no-foreign-land/current | tai64nlocal | tail -20" +echo "" diff --git a/dbus-no-foreign-land/install_gui.sh b/dbus-no-foreign-land/install_gui.sh new file mode 100644 index 0000000..26782ba --- /dev/null +++ b/dbus-no-foreign-land/install_gui.sh @@ -0,0 +1,163 @@ +#!/bin/bash +# +# Install GUI modifications for NFL (No Foreign Land) Tracking +# +# This script: +# 1. Copies PageSettingsNflTracking.qml to the GUI directory +# 2. Patches PageSettingsGps.qml to add NFL Tracking (inserts between Format and Speed Unit) +# 3. Restarts the GUI +# +# Note: These changes will be lost on firmware update. Run again after updating. +# +# Usage: +# ./install_gui.sh # Install GUI modifications +# ./install_gui.sh --remove # Remove GUI modifications and restore originals +# + +set -e + +QML_DIR="/opt/victronenergy/gui/qml" +INSTALL_DIR="/data/dbus-no-foreign-land" +BACKUP_DIR="/data/dbus-no-foreign-land/backup" + +install_gui() { + echo "==================================================" + echo "Installing NFL Tracking GUI modifications..." + echo "==================================================" + + # Check if our service is installed + if [ ! -f "$INSTALL_DIR/nfl_tracking.py" ]; then + echo "ERROR: Main service not installed. Run install.sh first." + exit 1 + fi + + # Check if QML directory exists + if [ ! -d "$QML_DIR" ]; then + echo "ERROR: GUI directory not found: $QML_DIR" + exit 1 + fi + + # Create backup directory + mkdir -p "$BACKUP_DIR" + + # Step 1: Copy our QML file + echo "1. Installing PageSettingsNflTracking.qml..." + if [ -f "$INSTALL_DIR/qml/PageSettingsNflTracking.qml" ]; then + cp "$INSTALL_DIR/qml/PageSettingsNflTracking.qml" "$QML_DIR/" + echo " Copied to $QML_DIR/" + else + echo "ERROR: PageSettingsNflTracking.qml not found" + exit 1 + fi + + # Step 2: Patch PageSettingsGps.qml (insert between Format and Speed Unit) + echo "2. Patching PageSettingsGps.qml..." + GPS_QML="$QML_DIR/PageSettingsGps.qml" + if [ ! -f "$GPS_QML" ]; then + echo " PageSettingsGps.qml not found - configure via Settings > GPS > NFL Tracking" + elif grep -q "PageSettingsNflTracking" "$GPS_QML"; then + echo " Already patched, skipping..." + else + if [ ! -f "$BACKUP_DIR/PageSettingsGps.qml.orig" ]; then + cp "$GPS_QML" "$BACKUP_DIR/PageSettingsGps.qml.orig" + echo " Backup saved to $BACKUP_DIR/" + fi + python3 << 'PYTHON_SCRIPT' +import re +qml_file = "/opt/victronenergy/gui/qml/PageSettingsGps.qml" +with open(qml_file, 'r') as f: + content = f.read() +if 'PageSettingsNflTracking' in content: + print("Already patched") + exit(0) +# Insert before Speed Unit block - exact structure from original file +menu_entry = ''' + MbSubMenu { + description: qsTr("NFL Tracking") + subpage: + Component { + PageSettingsNflTracking { + title: qsTr("NFL Tracking") + } + } + } + +''' +pattern = r"(\n\t\tMbItemOptions \{\n\t\t\tid: speedUnit)" +new_content = re.sub(pattern, menu_entry + r'\1', content, count=1) +if new_content == content: + pattern2 = r"(\n\t\tMbItemOptions \{\n\t\t\tid: format)" + new_content = re.sub(pattern2, menu_entry + r'\1', content, count=1) +if new_content == content: + print("ERROR: Could not find insertion point") + exit(1) +with open(qml_file, 'w') as f: + f.write(new_content) +print("Patched successfully") +PYTHON_SCRIPT + if [ $? -ne 0 ]; then exit 1; fi + echo " Patch applied" + fi + + # Step 3: Restart GUI + echo "3. Restarting GUI..." + if [ -d /service/start-gui ]; then + svc -t /service/start-gui + sleep 2 + elif [ -d /service/gui ]; then + svc -t /service/gui + sleep 2 + else + echo " Note: GUI service not found. Restart manually or reboot." + fi + + echo "" + echo "==================================================" + echo "GUI installation complete!" + echo "==================================================" + echo "" + echo "NFL Tracking should appear in: Settings -> GPS -> NFL Tracking" + echo "If GUI goes blank, run: install_gui.sh --remove" + echo "" + echo "Note: Run this script again after firmware updates." + echo "" +} + +remove_gui() { + echo "==================================================" + echo "Removing NFL Tracking GUI modifications..." + echo "==================================================" + + if [ -f "$QML_DIR/PageSettingsNflTracking.qml" ]; then + rm "$QML_DIR/PageSettingsNflTracking.qml" + echo "1. Removed PageSettingsNflTracking.qml" + fi + + if [ -f "$BACKUP_DIR/PageSettingsGps.qml.orig" ]; then + cp "$BACKUP_DIR/PageSettingsGps.qml.orig" "$QML_DIR/PageSettingsGps.qml" + echo "2. Restored original PageSettingsGps.qml" + fi + + echo "3. Restarting GUI..." + if [ -d /service/start-gui ]; then + svc -t /service/start-gui + sleep 2 + elif [ -d /service/gui ]; then + svc -t /service/gui + sleep 2 + fi + + echo "" + echo "GUI modifications removed." + echo "" +} + +# Main +case "$1" in + --remove) + remove_gui + ;; + *) + install_gui + ;; +esac diff --git a/dbus-no-foreign-land/nfl_tracking.py b/dbus-no-foreign-land/nfl_tracking.py new file mode 100644 index 0000000..7a23edb --- /dev/null +++ b/dbus-no-foreign-land/nfl_tracking.py @@ -0,0 +1,839 @@ +#!/usr/bin/env python3 +""" +NFL (No Foreign Land) Tracking for Venus OS + +Sends boat position/track to noforeignland.com using D-Bus GPS data. +Publishes its own D-Bus service so status is visible via MQTT: + N//nfltracking/0/... + +Features: +- Reads GPS position from Venus OS D-Bus +- Sends track data to the NFL API +- Settings adjustable via D-Bus/MQTT and Venus GUI +- 24h keepalive when boat is stationary +- Runs as a daemon with automatic restart + +Author: Adapted from dbus-generator-ramp style +License: MIT +""" + +import calendar +import json +import logging +import math +import os +import signal +import sys +import time +from pathlib import Path + +# Add velib_python to path (Venus OS standard location) +sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'ext', 'velib_python')) +sys.path.insert(1, '/opt/victronenergy/velib_python') + +try: + from gi.repository import GLib +except ImportError: + print("ERROR: GLib not available. This script must run on Venus OS.") + sys.exit(1) + +try: + import dbus + from dbus.mainloop.glib import DBusGMainLoop + from vedbus import VeDbusService + from settingsdevice import SettingsDevice +except ImportError as e: + print(f"ERROR: Required module not available: {e}") + print("This script must run on Venus OS.") + sys.exit(1) + +from config import ( + SERVICE_NAME, NFL_API_URL, NFL_PLUGIN_API_KEY, + TRACKING_CONFIG, CONFIG_DIR, TRACK_FILE, LOGGING_CONFIG, +) +import threading + +# Version +VERSION = '2.0.0' + +BUS_ITEM = "com.victronenergy.BusItem" +SYSTEM_SERVICE = "com.victronenergy.system" + + +def _unwrap(v): + """Minimal dbus value unwrap.""" + if v is None: + return None + if isinstance(v, (dbus.Int16, dbus.Int32, dbus.Int64, dbus.UInt16, dbus.UInt32, dbus.UInt64, dbus.Byte)): + return int(v) + if isinstance(v, dbus.Double): + return float(v) + if isinstance(v, (dbus.String, dbus.Signature)): + return str(v) + if isinstance(v, dbus.Boolean): + return bool(v) + if isinstance(v, dbus.Array): + if len(v) == 0: + return None + return [_unwrap(x) for x in v] + if isinstance(v, (dbus.Dictionary, dict)): + return {k: _unwrap(x) for k, x in v.items()} + return v + + +def validate_position(lat, lon): + """Validate lat/lon for WGS84.""" + try: + lat_f = float(lat) + lon_f = float(lon) + if not (-90 < lat_f < 90 and -180 < lon_f < 180): + return False + if abs(lat_f) < 0.01 and abs(lon_f) < 0.01: + return False # Likely GPS init + return True + except (TypeError, ValueError): + return False + + +def equirectangular_distance(lat1, lon1, lat2, lon2): + """Distance in meters (approximate).""" + R = 6371000 # Earth radius meters + phi1 = math.radians(lat1) + phi2 = math.radians(lat2) + dphi = math.radians(lat2 - lat1) + dlam = math.radians(lon2 - lon1) + x = dlam * math.cos((phi1 + phi2) / 2) + y = dphi + return math.sqrt(x * x + y * y) * R + + +class GpsReader: + """Read GPS position from Venus OS D-Bus.""" + + def __init__(self, bus): + self.bus = bus + self._gps_service = None + self._proxy_gps_svc = None + self._proxy_lat = None + self._proxy_lon = None + self._proxy_pos = None + self._proxy_fix = None + self._proxy_lat_nested = None + self._proxy_lon_nested = None + + def _get_proxy(self, service, path): + try: + obj = self.bus.get_object(service, path, introspect=False) + return dbus.Interface(obj, BUS_ITEM) + except dbus.exceptions.DBusException: + return None + + def _refresh_gps_service(self): + """Resolve GpsService to actual service name.""" + if self._gps_service: + return True + try: + proxy = self._get_proxy(SYSTEM_SERVICE, "/GpsService") + if proxy: + v = proxy.GetValue() + svc = _unwrap(v) + if svc and isinstance(svc, str): + self._gps_service = svc + self._proxy_lat = self._get_proxy(svc, "/Latitude") + self._proxy_lon = self._get_proxy(svc, "/Longitude") + self._proxy_pos = self._get_proxy(svc, "/Position") + self._proxy_fix = self._get_proxy(svc, "/Fix") + self._proxy_lat_nested = self._get_proxy(svc, "/Position/Latitude") + self._proxy_lon_nested = self._get_proxy(svc, "/Position/Longitude") + return True + except dbus.exceptions.DBusException: + pass + return False + + def get_position(self): + """Return (lat, lon) or None if no fix.""" + if not self._refresh_gps_service(): + return None + + if self._proxy_fix: + try: + v = self._proxy_fix.GetValue() + f = _unwrap(v) + if f is not None and not f: + return None + except dbus.exceptions.DBusException: + pass + + lat, lon = None, None + + if self._proxy_lat: + try: + lat = _unwrap(self._proxy_lat.GetValue()) + except dbus.exceptions.DBusException: + pass + if lat is None and self._proxy_lat_nested: + try: + lat = _unwrap(self._proxy_lat_nested.GetValue()) + except dbus.exceptions.DBusException: + pass + + if self._proxy_lon: + try: + lon = _unwrap(self._proxy_lon.GetValue()) + except dbus.exceptions.DBusException: + pass + if lon is None and self._proxy_lon_nested: + try: + lon = _unwrap(self._proxy_lon_nested.GetValue()) + except dbus.exceptions.DBusException: + pass + + if lat is not None and lon is not None: + try: + lat_f = float(lat) + lon_f = float(lon) + if -90 <= lat_f <= 90 and -180 <= lon_f <= 180: + return (lat_f, lon_f) + except (TypeError, ValueError): + pass + + if self._proxy_pos: + try: + v = self._proxy_pos.GetValue() + pos = _unwrap(v) + if isinstance(pos, str): + parts = pos.split(",") + if len(parts) >= 2: + lat_f = float(parts[0].strip()) + lon_f = float(parts[1].strip()) + if -90 <= lat_f <= 90 and -180 <= lon_f <= 180: + return (lat_f, lon_f) + except (dbus.exceptions.DBusException, TypeError, ValueError): + pass + + return None + + +def send_track_to_api(track, boat_api_key, log): + """POST track to NFL API. Returns True on success.""" + if not track: + return False + if not boat_api_key or not boat_api_key.strip(): + log.error("No boat API key configured") + return False + + import urllib.request + import urllib.error + import urllib.parse + + last_ts = track[-1][0] + params_dict = { + "timestamp": str(last_ts), + "track": json.dumps(track), + "boatApiKey": boat_api_key.strip(), + } + params = urllib.parse.urlencode(params_dict).encode() + + headers = { + "X-NFL-API-Key": NFL_PLUGIN_API_KEY, + "Content-Type": "application/x-www-form-urlencoded", + "User-Agent": "dbus-no-foreign-land/2.0 (Venus OS)", + } + + req = urllib.request.Request( + NFL_API_URL, + data=params, + headers=headers, + method="POST", + ) + + # Debug: log full request (mask boatApiKey) + if log.isEnabledFor(logging.DEBUG): + debug_params = dict(params_dict) + key = boat_api_key.strip() + debug_params["boatApiKey"] = key[:4] + "***" + key[-4:] if len(key) > 8 else "***" + log.debug("API request URL: %s", NFL_API_URL) + log.debug("API request headers: %s", headers) + log.debug("API request body: %s", urllib.parse.urlencode(debug_params)) + + try: + with urllib.request.urlopen(req, timeout=30) as resp: + body = json.loads(resp.read().decode()) + if body.get("status") == "ok": + log.info("Track sent successfully (%d points)", len(track)) + return True + log.warning("API error: %s", body.get("message", "Unknown")) + return False + except urllib.error.HTTPError as e: + err_body = "" + try: + err_body = e.read().decode() + except Exception as read_err: + log.warning("Could not read error body: %s", read_err) + err_msg = e.reason + if err_body: + try: + err_json = json.loads(err_body) + err_msg = err_json.get("message", err_body[:300]) + except json.JSONDecodeError: + err_msg = err_body[:300] if err_body else e.reason + log.error("HTTP %d: %s - %s", e.code, e.reason, err_msg) + # On 403, always log what we sent (masked) to help debug + if e.code == 403: + key = boat_api_key.strip() + masked_key = key[:4] + "***" + key[-4:] if len(key) > 8 else "***" + log.error("Request was: URL=%s | headers=%s | body: timestamp=%s track_len=%d boatApiKey=%s", + NFL_API_URL, headers, params_dict["timestamp"], len(track), masked_key) + return False + except Exception as e: + log.error("Send failed: %s", e) + return False + + +def load_track_from_file(path, max_points=None): + """Load track points from JSONL file. Returns list of [timestamp_ms, lat, lon]. + If max_points is set, returns only the most recent points (keeps tail). + """ + max_points = max_points or TRACKING_CONFIG.get('max_track_points', 10000) + track = [] + p = Path(path) + if not p.exists(): + return track + try: + with open(p) as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + st = time.strptime(obj["t"], "%Y-%m-%dT%H:%M:%SZ") + ts = int(calendar.timegm(st) * 1000) + lat, lon = float(obj["lat"]), float(obj["lon"]) + if validate_position(lat, lon): + track.append([ts, lat, lon]) + except (KeyError, ValueError, TypeError): + pass + if len(track) > max_points: + track = track[-max_points:] + except OSError: + pass + return track + + +class NflTrackingController: + """ + Main controller that coordinates: + - D-Bus monitoring of GPS position + - Track logging and API upload + - Publishing status to D-Bus (visible via MQTT) + """ + + def __init__(self): + self._setup_logging() + self.logger = logging.getLogger('NflTracking') + self.logger.info(f"Initializing NFL Tracking v{VERSION}") + + # State + self.track = [] + self._send_in_progress = False + self.last_pos = None + self.last_log_time = 0 + self.last_send_time = time.time() + self.last_send_time_success = 0 # Unix timestamp of last successful send + self.last_point_time = time.time() + + # D-Bus connection + self.bus = dbus.SystemBus() + + # Create our D-Bus service for publishing status + self._create_dbus_service() + + # Set up settings (stored in Venus localsettings) + self._setup_settings() + + # GPS reader + self.gps = GpsReader(self.bus) + + # Load existing track (capped to prevent OOM) + self.track = load_track_from_file(TRACK_FILE) + self._max_track_points = TRACKING_CONFIG.get('max_track_points', 10000) + + # Ensure config is set (from _load_settings or defaults) + if not hasattr(self, 'boat_api_key'): + self.boat_api_key = '' + self.boat_name = '' + self.enabled = True + self.track_interval = TRACKING_CONFIG['track_interval'] + self.min_distance = TRACKING_CONFIG['min_distance'] + self.send_interval = TRACKING_CONFIG['send_interval'] + self.ping_24h = TRACKING_CONFIG['ping_24h'] + + poll_interval_ms = min(30000, self.track_interval * 1000) + GLib.timeout_add(poll_interval_ms, self._main_loop) + + self.logger.info( + f"Initialized. Poll interval: {poll_interval_ms}ms, " + f"track_interval={self.track_interval}s, send_interval={self.send_interval}min" + ) + + def _setup_logging(self): + """Configure logging based on config.""" + level = getattr(logging, LOGGING_CONFIG['level'], logging.INFO) + if LOGGING_CONFIG['include_timestamp']: + fmt = '%(asctime)s %(levelname)s %(name)s: %(message)s' + else: + fmt = '%(levelname)s %(name)s: %(message)s' + logging.basicConfig(level=level, format=fmt, stream=sys.stdout) + + def _create_dbus_service(self): + """Create our own D-Bus service for publishing status.""" + self.logger.info(f"Creating D-Bus service: {SERVICE_NAME}") + + max_retries = 5 + retry_delay = 1.0 + + for attempt in range(max_retries): + try: + # register=False: add all paths first, then call register() + # See https://github.com/victronenergy/venus/wiki/dbus-api + self.dbus_service = VeDbusService(SERVICE_NAME, self.bus, register=False) + break + except dbus.exceptions.NameExistsException: + if attempt < max_retries - 1: + self.logger.warning( + f"D-Bus name exists, retrying in {retry_delay}s " + f"(attempt {attempt + 1}/{max_retries})" + ) + time.sleep(retry_delay) + retry_delay *= 2 + else: + self.logger.error("Failed to acquire D-Bus name after retries") + raise + + # Management paths (required for Venus) + self.dbus_service.add_path('/Mgmt/ProcessName', 'dbus-no-foreign-land') + self.dbus_service.add_path('/Mgmt/ProcessVersion', VERSION) + self.dbus_service.add_path('/Mgmt/Connection', 'local') + + # Device info + self.dbus_service.add_path('/DeviceInstance', 0) + self.dbus_service.add_path('/ProductId', 0xFFFF) + self.dbus_service.add_path('/ProductName', 'NFL Tracking') + self.dbus_service.add_path('/FirmwareVersion', VERSION) + self.dbus_service.add_path('/Connected', 1) + + # Status paths (read-only) + def _status_text(p, v): + if v is not None and v in (0, 1, 2): + return ['Idle', 'Tracking', 'Sending'][v] + return 'Unknown' + self.dbus_service.add_path('/Status', 0, gettextcallback=_status_text) + self.dbus_service.add_path('/TrackPoints', 0) + self.dbus_service.add_path('/Stationary', 0) + self.dbus_service.add_path('/LastSendTimeAgo', '') + def _coord_text(p, v): + if v is not None and isinstance(v, (int, float)): + return f"{float(v):.6f}" + return "--" + self.dbus_service.add_path('/LastLatitude', 0.0, gettextcallback=_coord_text) + self.dbus_service.add_path('/LastLongitude', 0.0, gettextcallback=_coord_text) + self.dbus_service.add_path('/LastUpdate', 0) + self.dbus_service.add_path('/GpsConnected', 0) + self.dbus_service.add_path('/LastSendTime', 0, + gettextcallback=self._last_send_time_text) + + # Action: send now (writable, 0->1 triggers send) + # Under /Settings/ so Venus GUI/MQTT exposes it for the switch + self.dbus_service.add_path('/Settings/SendNow', 0, + writeable=True, + onchangecallback=self._on_send_now) + + # Writable settings (can be changed via D-Bus/MQTT) + self.dbus_service.add_path('/Settings/BoatApiKey', '', + writeable=True, + onchangecallback=self._on_setting_changed) + self.dbus_service.add_path('/Settings/BoatName', '', + writeable=True, + onchangecallback=self._on_setting_changed) + self.dbus_service.add_path('/Settings/Enabled', 1, + writeable=True, + onchangecallback=self._on_setting_changed) + def _int_text(p, v, suffix): + return f"{v} {suffix}" if v is not None else "--" + self.dbus_service.add_path('/Settings/TrackInterval', TRACKING_CONFIG['track_interval'], + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: _int_text(p, v, "s")) + self.dbus_service.add_path('/Settings/MinDistance', TRACKING_CONFIG['min_distance'], + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: _int_text(p, v, "m")) + self.dbus_service.add_path('/Settings/SendInterval', TRACKING_CONFIG['send_interval'], + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: _int_text(p, v, "min")) + self.dbus_service.add_path('/Settings/Ping24h', 1 if TRACKING_CONFIG['ping_24h'] else 0, + writeable=True, + onchangecallback=self._on_setting_changed) + + # Register after all paths added (required by Venus D-Bus API) + self.dbus_service.register() + self.logger.info("D-Bus service created") + + def _update_last_send_time_ago(self): + """Update /LastSendTimeAgo with Xh Ym format.""" + if self.last_send_time_success <= 0: + self.dbus_service['/LastSendTimeAgo'] = 'Never' + else: + diff_sec = time.time() - self.last_send_time_success + total_m = int(diff_sec / 60) + h, m = total_m // 60, total_m % 60 + self.dbus_service['/LastSendTimeAgo'] = f"{h}h {m}m" + + def _last_send_time_text(self, path, value): + """Format last send time for display.""" + try: + if value is None or (isinstance(value, (int, float)) and value <= 0): + return "Never" + val = float(value) + except (TypeError, ValueError): + return "Never" + now = time.time() + diff = now - val + if diff < 60: + return "Just now" + if diff < 3600: + return f"{int(diff / 60)} min ago" + if diff < 86400: + return f"{int(diff / 3600)} h ago" + return time.strftime("%Y-%m-%d %H:%M", time.localtime(val)) + + def _on_send_now(self, path, value): + """Handle 'Send now' action - trigger immediate send.""" + if not value: + return True + self.logger.info("Send now requested via D-Bus/MQTT") + try: + self._do_send_now() + finally: + # Always reset switch; defer slightly so GUI/MQTT can process + def _reset(): + self.dbus_service['/Settings/SendNow'] = 0 + return False # one-shot + GLib.timeout_add(300, _reset) + return True + + def _send_track_async(self, to_send, on_done): + """Send track in background thread; on_done(ok) runs on main thread via GLib.idle_add.""" + def _worker(): + ok = send_track_to_api(to_send, self.boat_api_key, self.logger) + GLib.idle_add(lambda: on_done(ok)) + + t = threading.Thread(target=_worker, daemon=True) + t.start() + + def _do_send_now(self): + """Perform immediate track send to API (non-blocking).""" + if self._send_in_progress: + self.logger.warning("Send already in progress, skipping") + return + to_send = load_track_from_file(TRACK_FILE) if Path(TRACK_FILE).exists() else self.track + if not to_send: + self.logger.warning("No track points to send") + return + self._send_in_progress = True + self.dbus_service['/Status'] = 2 # Sending + + def _on_done(ok): + self._send_in_progress = False + if ok: + self.last_send_time = time.time() + self.last_send_time_success = int(time.time()) + self.dbus_service['/LastSendTime'] = self.last_send_time_success + self.track.clear() + try: + if Path(TRACK_FILE).exists(): + Path(TRACK_FILE).unlink() + except OSError: + pass + self.dbus_service['/Status'] = 1 # Back to Tracking + + self._send_track_async(to_send, _on_done) + + def _on_setting_changed(self, path, value): + """Handle setting changes from D-Bus/MQTT.""" + self.logger.info(f"Setting changed: {path} = {value}") + + if path == '/Settings/BoatApiKey': + self.boat_api_key = str(value) if value is not None else '' + self._save_setting('BoatApiKey', self.boat_api_key) + + elif path == '/Settings/BoatName': + self.boat_name = str(value) if value is not None else '' + self._save_setting('BoatName', self.boat_name) + + elif path == '/Settings/Enabled': + self.enabled = bool(value) + self._save_setting('Enabled', 1 if self.enabled else 0) + + elif path == '/Settings/TrackInterval': + val = int(value) if value is not None else TRACKING_CONFIG['track_interval'] + self.track_interval = max(10, min(3600, val)) + self._save_setting('TrackInterval', self.track_interval) + + elif path == '/Settings/MinDistance': + val = int(value) if value is not None else TRACKING_CONFIG['min_distance'] + self.min_distance = max(1, min(10000, val)) + self._save_setting('MinDistance', self.min_distance) + + elif path == '/Settings/SendInterval': + val = int(value) if value is not None else TRACKING_CONFIG['send_interval'] + self.send_interval = max(1, min(1440, val)) + self._save_setting('SendInterval', self.send_interval) + + elif path == '/Settings/Ping24h': + self.ping_24h = bool(value) + self._save_setting('Ping24h', 1 if self.ping_24h else 0) + + return True + + def _save_setting(self, name, value): + """Save a setting to localsettings.""" + if self.settings: + try: + self.settings[name] = value + except Exception as e: + self.logger.warning(f"Failed to save setting {name}: {e}") + + def _setup_settings(self): + """Set up persistent settings via Venus localsettings.""" + self.settings = None + try: + settings_path = '/Settings/NflTracking' + + settings_def = { + 'BoatApiKey': [settings_path + '/BoatApiKey', '', 0, 0], + 'BoatName': [settings_path + '/BoatName', '', 0, 0], + 'Enabled': [settings_path + '/Enabled', 1, 0, 1], + 'TrackInterval': [settings_path + '/TrackInterval', TRACKING_CONFIG['track_interval'], 10, 3600], + 'MinDistance': [settings_path + '/MinDistance', TRACKING_CONFIG['min_distance'], 1, 10000], + 'SendInterval': [settings_path + '/SendInterval', TRACKING_CONFIG['send_interval'], 1, 1440], + 'Ping24h': [settings_path + '/Ping24h', 1 if TRACKING_CONFIG['ping_24h'] else 0, 0, 1], + } + + self.settings = SettingsDevice( + self.bus, + settings_def, + self._on_persistent_setting_changed + ) + + if self.settings: + self._load_settings() + + self.logger.info("Persistent settings initialized") + + except Exception as e: + self.logger.warning(f"Could not initialize persistent settings: {e}") + self.logger.warning("Settings will not persist across restarts") + + def _load_settings(self): + """Load settings from Venus localsettings.""" + if not self.settings: + return + + try: + boat_api_key = str(self.settings['BoatApiKey']) if self.settings['BoatApiKey'] else '' + boat_name = str(self.settings['BoatName']) if self.settings['BoatName'] else '' + enabled = bool(self.settings['Enabled']) + track_interval = int(self.settings['TrackInterval']) + min_distance = int(self.settings['MinDistance']) + send_interval = int(self.settings['SendInterval']) + ping_24h = bool(self.settings['Ping24h']) + + self.track_interval = max(10, min(3600, track_interval)) + self.min_distance = max(1, min(10000, min_distance)) + self.send_interval = max(1, min(1440, send_interval)) + self.ping_24h = ping_24h + self.boat_api_key = boat_api_key + self.boat_name = boat_name + self.enabled = enabled + + self.dbus_service['/Settings/BoatApiKey'] = boat_api_key + self.dbus_service['/Settings/BoatName'] = boat_name + self.dbus_service['/Settings/Enabled'] = 1 if enabled else 0 + self.dbus_service['/Settings/TrackInterval'] = self.track_interval + self.dbus_service['/Settings/MinDistance'] = self.min_distance + self.dbus_service['/Settings/SendInterval'] = self.send_interval + self.dbus_service['/Settings/Ping24h'] = 1 if ping_24h else 0 + + self.logger.info( + f"Loaded settings: boat_api_key={'*' * 8 if boat_api_key else '(empty)'}, " + f"track_interval={self.track_interval}s, send_interval={self.send_interval}min" + ) + + except Exception as e: + self.logger.warning(f"Error loading settings: {e}") + + def _on_persistent_setting_changed(self, setting, old_value, new_value): + """Called when a persistent setting changes externally.""" + self.logger.info(f"Persistent setting changed: {setting} = {new_value}") + self._load_settings() + + def _main_loop(self) -> bool: + """Main control loop - called periodically by GLib.""" + try: + if not self.enabled: + self.dbus_service['/Status'] = 0 # Idle + self.dbus_service['/Stationary'] = 0 + self._update_last_send_time_ago() + return True + + self.dbus_service['/Status'] = 1 # Tracking + + # Read GPS position + pos = self.gps.get_position() + self.dbus_service['/GpsConnected'] = 1 if pos else 0 + + if pos: + lat, lon = pos + if validate_position(lat, lon): + now_ms = int(time.time() * 1000) + now = time.time() + + self.dbus_service['/LastLatitude'] = lat + self.dbus_service['/LastLongitude'] = lon + self.dbus_service['/LastUpdate'] = now_ms + + force_save = ( + self.ping_24h and self.last_point_time and + (now - self.last_point_time) >= 24 * 3600 + ) + if force_save or not self.last_pos: + do_log = True + elif now - self.last_log_time >= self.track_interval: + dist = equirectangular_distance( + self.last_pos[0], self.last_pos[1], lat, lon + ) + do_log = dist >= self.min_distance + else: + do_log = False + + # Stationary = has last_pos, hasn't moved min_distance + if self.last_pos and not do_log and not force_save: + self.dbus_service['/Stationary'] = 1 + else: + self.dbus_service['/Stationary'] = 0 + + if do_log: + self.track.append([now_ms, lat, lon]) + trimmed = False + if len(self.track) > self._max_track_points: + self.track = self.track[-self._max_track_points:] + # Trim file to match (prevents unbounded disk growth) + try: + Path(CONFIG_DIR).mkdir(parents=True, exist_ok=True) + with open(TRACK_FILE, 'w') as f: + for pt in self.track: + ts_ms, lat_pt, lon_pt = pt + t_str = time.strftime( + "%Y-%m-%dT%H:%M:%SZ", time.gmtime(ts_ms / 1000) + ) + f.write(json.dumps({"t": t_str, "lat": lat_pt, "lon": lon_pt}) + "\n") + trimmed = True + except OSError: + pass + self.last_pos = (lat, lon) + self.last_log_time = now + self.last_point_time = now + if not trimmed: + try: + Path(CONFIG_DIR).mkdir(parents=True, exist_ok=True) + with open(TRACK_FILE, "a") as f: + f.write(json.dumps({ + "t": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now)), + "lat": lat, "lon": lon + }) + "\n") + except OSError: + pass + self.logger.debug("Logged point: %.6f, %.6f", lat, lon) + + # Maybe send (non-blocking; skip if send already in progress) + now = time.time() + if not self._send_in_progress and now - self.last_send_time >= self.send_interval * 60: + to_send = load_track_from_file(TRACK_FILE) if Path(TRACK_FILE).exists() else self.track + if to_send: + self._send_in_progress = True + self.dbus_service['/Status'] = 2 # Sending + + def _on_done(ok): + self._send_in_progress = False + if ok: + self.last_send_time_success = int(time.time()) + self.dbus_service['/LastSendTime'] = self.last_send_time_success + self.track.clear() + try: + if Path(TRACK_FILE).exists(): + Path(TRACK_FILE).unlink() + except OSError: + pass + self.last_send_time = time.time() + self.dbus_service['/Status'] = 1 # Back to Tracking + + self._send_track_async(to_send, _on_done) + else: + self.last_send_time = now + + self.dbus_service['/TrackPoints'] = len(self.track) + self._update_last_send_time_ago() + + # Update stationary when no GPS (not stationary, unknown) + if not pos: + self.dbus_service['/Stationary'] = 0 + + except dbus.exceptions.DBusException as e: + self.logger.warning("D-Bus error: %s", e) + except Exception as e: + self.logger.exception("Unexpected error: %s", e) + + return True + + +def main(): + """Main entry point.""" + DBusGMainLoop(set_as_default=True) + + print("=" * 60) + print(f"NFL Tracking v{VERSION}") + print("=" * 60) + + mainloop = None + + def signal_handler(signum, frame): + """Handle shutdown signals gracefully.""" + try: + sig_name = signal.Signals(signum).name + except ValueError: + sig_name = str(signum) + logging.info(f"Received {sig_name}, shutting down...") + if mainloop is not None: + mainloop.quit() + + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + + try: + controller = NflTrackingController() + mainloop = GLib.MainLoop() + mainloop.run() + except KeyboardInterrupt: + print("\nShutdown requested") + except Exception as e: + logging.error(f"Fatal error: {e}", exc_info=True) + sys.exit(1) + finally: + logging.info("Service stopped") + + +if __name__ == '__main__': + main() diff --git a/dbus-no-foreign-land/qml/PageSettingsNflTracking.qml b/dbus-no-foreign-land/qml/PageSettingsNflTracking.qml new file mode 100644 index 0000000..dc2c7ce --- /dev/null +++ b/dbus-no-foreign-land/qml/PageSettingsNflTracking.qml @@ -0,0 +1,101 @@ +import QtQuick 2 +import com.victron.velib 1.0 +import "utils.js" as Utils + +MbPage { + id: root + title: qsTr("NFL Tracking") + property string bindPrefix: "com.victronenergy.nfltracking" + property VBusItem connectedItem: VBusItem { bind: Utils.path(bindPrefix, "/Connected") } + property VBusItem trackPointsItem: VBusItem { bind: Utils.path(bindPrefix, "/TrackPoints") } + property VBusItem lastSendTimeAgoItem: VBusItem { bind: Utils.path(bindPrefix, "/LastSendTimeAgo") } + property VBusItem stationaryItem: VBusItem { bind: Utils.path(bindPrefix, "/Stationary") } + + model: VisibleItemModel { + MbItemText { + text: qsTr("Service not running - check installation") + show: !connectedItem.valid + } + + MbItemText { + text: qsTr("Pending: %1 Last reported: %2").arg( + trackPointsItem.valid ? trackPointsItem.value : "--").arg( + lastSendTimeAgoItem.valid ? lastSendTimeAgoItem.value : "--") + show: connectedItem.valid + } + + MbItemText { + text: stationaryItem.valid && stationaryItem.value ? qsTr("Vessel stationary (has not moved min distance)") : qsTr("Vessel moving") + show: connectedItem.valid + } + + MbEditBox { + description: qsTr("Boat API Key") + show: connectedItem.valid + item { + bind: Utils.path(bindPrefix, "/Settings/BoatApiKey") + } + } + + MbEditBox { + description: qsTr("Boat name") + show: connectedItem.valid + item { + bind: Utils.path(bindPrefix, "/Settings/BoatName") + } + } + + MbSwitch { + id: enableSwitch + name: qsTr("Enable tracking") + bind: Utils.path(bindPrefix, "/Settings/Enabled") + enabled: connectedItem.valid + show: connectedItem.valid + } + + MbSpinBox { + description: qsTr("Track interval (min between points)") + show: connectedItem.valid && enableSwitch.checked + item { + bind: Utils.path(bindPrefix, "/Settings/TrackInterval") + unit: "s" + decimals: 0 + step: 10 + min: 10 + max: 3600 + } + } + + MbSpinBox { + description: qsTr("Min distance before new point") + show: connectedItem.valid && enableSwitch.checked + item { + bind: Utils.path(bindPrefix, "/Settings/MinDistance") + unit: "m" + decimals: 0 + step: 10 + min: 1 + max: 10000 + } + } + + MbSpinBox { + description: qsTr("Send interval") + show: connectedItem.valid && enableSwitch.checked + item { + bind: Utils.path(bindPrefix, "/Settings/SendInterval") + unit: "min" + decimals: 0 + step: 5 + min: 1 + max: 1440 + } + } + + MbSwitch { + name: qsTr("24h keepalive when stationary") + bind: Utils.path(bindPrefix, "/Settings/Ping24h") + show: connectedItem.valid && enableSwitch.checked + } + } +} diff --git a/dbus-no-foreign-land/restart.sh b/dbus-no-foreign-land/restart.sh new file mode 100644 index 0000000..e3ccb16 --- /dev/null +++ b/dbus-no-foreign-land/restart.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Restart NFL Tracking service + +SERVICE_DIR="" +if [ -d "/service/dbus-no-foreign-land" ]; then + SERVICE_DIR="/service/dbus-no-foreign-land" +elif [ -d "/opt/victronenergy/service/dbus-no-foreign-land" ]; then + SERVICE_DIR="/opt/victronenergy/service/dbus-no-foreign-land" +else + echo "Service not found" + exit 1 +fi + +svc -t "$SERVICE_DIR" +echo "Restarted NFL Tracking" diff --git a/dbus-no-foreign-land/service/log/run b/dbus-no-foreign-land/service/log/run new file mode 100644 index 0000000..6bf039e --- /dev/null +++ b/dbus-no-foreign-land/service/log/run @@ -0,0 +1,8 @@ +#!/bin/sh +# +# daemontools log run script +# +# Logs service output using multilog +# + +exec multilog t s99999 n8 /var/log/dbus-no-foreign-land diff --git a/dbus-no-foreign-land/service/run b/dbus-no-foreign-land/service/run new file mode 100644 index 0000000..8b8d440 --- /dev/null +++ b/dbus-no-foreign-land/service/run @@ -0,0 +1,18 @@ +#!/bin/sh +# +# daemontools run script for NFL (No Foreign Land) Tracking +# +# This script is called by daemontools to start the service. +# It will be automatically restarted if it exits. +# + +exec 2>&1 + +# Change to the script directory +cd /data/dbus-no-foreign-land + +# Add velib_python to path (use the symlink created by install.sh) +export PYTHONPATH="/data/dbus-no-foreign-land/ext/velib_python:$PYTHONPATH" + +# Execute the main script +exec python3 /data/dbus-no-foreign-land/nfl_tracking.py diff --git a/dbus-no-foreign-land/uninstall.sh b/dbus-no-foreign-land/uninstall.sh new file mode 100644 index 0000000..cc8eda6 --- /dev/null +++ b/dbus-no-foreign-land/uninstall.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Uninstall NFL Tracking for Venus OS + +INSTALL_DIR="/data/dbus-no-foreign-land" +SERVICE_LINK="dbus-no-foreign-land" + +# Find service directory (matches install.sh logic) +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "Uninstalling NFL Tracking..." + +if [ -L "$SERVICE_DIR/$SERVICE_LINK" ] || [ -e "$SERVICE_DIR/$SERVICE_LINK" ]; then + echo "Stopping and removing service..." + svc -d "$SERVICE_DIR/$SERVICE_LINK" 2>/dev/null || true + rm -f "$SERVICE_DIR/$SERVICE_LINK" + rm -rf "$SERVICE_DIR/$SERVICE_LINK" +fi + +echo "Service removed. Config and data in $INSTALL_DIR are preserved." +echo "To remove everything: rm -rf $INSTALL_DIR /var/log/dbus-no-foreign-land" diff --git a/dbus-tides/.gitignore b/dbus-tides/.gitignore new file mode 100644 index 0000000..ac038fd --- /dev/null +++ b/dbus-tides/.gitignore @@ -0,0 +1,27 @@ +# Build artifacts +*.tar.gz +*.sha256 + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg +*.egg-info/ +dist/ +build/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Venus OS runtime (created during installation) +ext/ + +# Runtime databases +*.db diff --git a/dbus-tides/README.md b/dbus-tides/README.md new file mode 100644 index 0000000..dea7483 --- /dev/null +++ b/dbus-tides/README.md @@ -0,0 +1,377 @@ +# dbus-tides + +A Venus OS service that provides real-time tide predictions by combining +onboard depth-sensor observations with harmonic tidal models. It publishes +tide data on D-Bus (and therefore MQTT) so that marine displays, MFDs, and +other Venus OS consumers can show current and upcoming tide information. + +**Version:** 1.0.19 + +## How It Works + +### Overview + +The service continuously reads the vessel's depth sounder, detects observed +high and low tide events, and blends them with a harmonic tidal model to +produce an accurate, location-aware tide prediction curve. The result is +published as a set of D-Bus paths under `com.victronenergy.tides`. + +``` +Depth Sensor (D-Bus) + │ + ▼ + DepthRecorder Collect, average, persist 5-min depth samples + │ + ▼ + TideDetector Detect observed high/low tide turning points + │ + ▼ + TidePredictor Harmonic model prediction (NOAA stations or grid) + │ + ▼ + TideMerger Calibrate predictions to depth under keel + │ + ▼ + TideAdapter Local adaptation (timing offset, amplitude scaling) + │ + ▼ + D-Bus / MQTT Publish predictions, curves, and event slots +``` + +### Depth Collection (`depth_recorder.py`) + +Raw depth readings are sampled every 10 seconds. Readings are averaged over +5-minute windows to suppress wave and wake noise, then stored in a SQLite +database (`/data/dbus-tides/depth_history.db`) along with GPS coordinates. +Up to 96 hours of history is retained. + +When retrieving history for calibration or detection, records are filtered +by position so that only data from the current anchorage (within the +stationary radius) is used. + +### Tide Detection (`tide_detector.py`) + +The smoothed depth history is processed to identify turning points (high and +low tides). A two-stage approach is used: + +1. **Median filter** (kernel size 5) removes impulse noise from anchor swing. +2. **Triangular-weighted moving average** (60-minute window) produces a smooth + trend curve. +3. **Slope reversal detection** identifies candidate extrema. +4. **Amplitude gating** requires at least 0.4 m of depth change between + consecutive highs and lows (`MIN_TIDE_AMPLITUDE`). +5. **Confirmation** requires a 0.2 m excursion from the candidate before the + event is reported, preventing premature triggering on flat tides. + +### Harmonic Prediction (`tide_harmonics.py`, `tide_predictor.py`) + +Tide heights are computed using the standard IHO/Schureman harmonic method +with 37 tidal constituents: + +``` +h(t) = Z₀ + Σ fᵢ · Aᵢ · cos(Vᵢ(t) + uᵢ − Gᵢ) +``` + +Where: +- **Z₀** — mean water level +- **Aᵢ, Gᵢ** — amplitude and phase lag for each constituent +- **fᵢ, uᵢ** — nodal amplitude and phase corrections (18.6-year lunar cycle) +- **Vᵢ(t)** — astronomical argument computed from Doodson numbers + +#### Constituent list (37) + +| Category | Constituents | +|---|---| +| Semi-diurnal | M2, S2, N2, K2, 2N2, MU2, NU2, L2, T2, R2, LAM2 | +| Diurnal | K1, O1, P1, Q1, J1, OO1, M1, 2Q1, RHO1 | +| Long-period | MF, MM, SSA, MSM, MS | +| Shallow-water | M3, M4, MS4, M6, M8, MN4, S4, S6, 2SM2, MKS2 | +| Terdiurnal | MK3, 2MK3 | +| Other | S1, SA | + +The engine is pure Python with no external dependencies at runtime. + +#### Extrema detection + +Predicted tide events (highs and lows) are extracted from the harmonic curve +using a multi-pass algorithm: + +1. Identify all local maxima and minima. +2. Merge consecutive same-type extrema to enforce strict alternation. +3. Iteratively remove the adjacent pair with the smallest height range until + all remaining pairs exceed the minimum amplitude threshold. + +This avoids the cascading-rejection problem that single-pass filtering causes +when diurnal inequality produces small-range tidal cycles (common in the +Caribbean, Gulf of Mexico, and other mixed-tide regions). + +### Station Selection (`tide_predictor.py`) + +Constituent data can come from three sources, tried in priority order: + +1. **Manual override** — user specifies a NOAA station ID via + `/Settings/StationOverride`. +2. **Best-fit scoring** — if the vessel has accumulated at least 12 hours of + depth history, nearby NOAA stations are hindcast and correlated against + detrended observations. The station with the highest Pearson correlation + (minimum 0.3) wins. +3. **Nearest station** — falls back to the closest NOAA station within 50 nm. +4. **Coastal grid** — if no nearby station exists, constituents are + bilinearly interpolated from a pre-built 0.25° coastal grid (derived from + the GOT4.10c global ocean tide model). + +Up to 5 nearby NOAA stations are reported on `/Tide/NearbyStations` for the +UI to offer as manual selections. + +#### Subordinate stations + +NOAA subordinate stations (type `S`) do not have their own harmonic +constituents. Instead, predictions are derived from a reference harmonic +station by: + +1. Predicting the reference station's full curve. +2. Applying time offsets (`time_high`, `time_low`) interpolated between + reference extrema. +3. Applying height correction factors (`height_high`, `height_low`) as either + ratios or additive offsets, interpolated between reference extrema. + +### Chart Depth Calibration (`tide_merger.py`) + +Harmonic models predict heights relative to a tidal datum (typically mean sea +level). The depth sounder reports depth under the keel. The **chart depth +offset** bridges the two: + +``` +chart_depth = mean(observed_depth) − mean(predicted_tide_height) +``` + +This offset is computed from recent observations and converts model +predictions into expected depth under the keel. Once calibrated, the +predicted curve represents what the sounder should read at each point in time. + +### Position-Aware Recalibration + +When the vessel moves, the service responds at three distance thresholds: + +| Threshold | Distance | Action | +|---|---|---| +| `CHART_DEPTH_RECAL_METERS` | 500 ft (152 m) | Recalibrate chart depth using the last 30 minutes of observations at the new position | +| `STATIONARY_RADIUS_METERS` | 5 nm (9.3 km) | Depth history is filtered to the current anchorage | +| `MODEL_RERUN_RADIUS_METERS` | 20 nm (37 km) | Re-interpolate tidal constituents and re-run the harmonic model | + +On service restart, if no prior calibration position is known, calibration +uses only recent observations (last 30 minutes) to avoid contamination from +depth records at a previous anchorage. + +A speed gate (2 knots) suppresses depth readings while underway, since +vessel motion invalidates sounder accuracy. + +### Local Adaptation (`tide_adapter.py`) + +Even with the best available constituents, local geography (bays, inlets, +shallow banks) causes tides to arrive earlier or later and with different +amplitude compared to the open-ocean model. The adapter estimates two +correction factors from observed tide events matched against predictions: + +- **Δt** — timing offset in seconds (positive = local tide arrives later) +- **k** — amplitude scale factor (1.0 = no scaling) + +The corrected local prediction is: + +``` +local_height(t) = k · predicted_height(t − Δt) +``` + +Adaptation progresses through stages: +- **Status 0** — no observed events matched yet +- **Status 1** — timing correction only (at least 2 matched events) +- **Status 2** — timing + amplitude correction (at least 1 high/low range pair) + +#### Residual correction + +After applying timing and amplitude adjustments, a smoothed residual +(observed minus adapted model) is computed over a 1.5-hour window and added +to the prediction. Beyond the last observation, the residual correction +decays exponentially with a 6-hour half-life. + +### Dual-Curve Output + +The service publishes two prediction curves: +- **Station curve** — raw harmonic model shifted to depth under keel +- **Local curve** — station curve with timing, amplitude, and residual + corrections applied + +The UI can display either or both. The "best" curve (local when adaptation +is active, otherwise station) is published on `/Tide/Predictions`. + +## D-Bus / MQTT Topics + +All paths are published under service name `com.victronenergy.tides`. +On MQTT, paths map to topics prefixed with `N//tides/`. + +### Service Management + +| Path | Type | Description | +|---|---|---| +| `/Mgmt/ProcessName` | string | `dbus-tides` | +| `/Mgmt/ProcessVersion` | string | Service version | +| `/Mgmt/Connection` | string | `local` | +| `/DeviceInstance` | int | `0` | +| `/ProductId` | int | `0xA162` | +| `/ProductName` | string | `Tide Prediction` | +| `/FirmwareVersion` | string | Service version | +| `/Connected` | int | `1` | + +### Status + +| Path | Type | Description | +|---|---|---| +| `/Status` | int | `0` Idle, `1` Calibrating, `2` Ready, `3` Error | +| `/ErrorMessage` | string | Error description (empty when status is not 3) | +| `/IsStationary` | int | `1` if vessel is within the stationary radius | + +### Depth + +| Path | Type | Description | +|---|---|---| +| `/Depth/Current` | float | Latest smoothed depth in meters | +| `/Depth/History` | string | JSON array of `{"ts": , "depth": }` (last 24 h) | +| `/Depth/FullHistory` | string | JSON array of all observed depth points | + +### Predictions + +| Path | Type | Description | +|---|---|---| +| `/Tide/Predictions` | string | JSON array — best available prediction curve | +| `/Tide/Station/Predictions` | string | JSON array — raw station model curve | +| `/Tide/Local/Predictions` | string | JSON array — locally adapted curve | + +### Local Adaptation + +| Path | Type | Description | +|---|---|---| +| `/Tide/Local/TimeOffset` | float | Timing offset Δt in seconds | +| `/Tide/Local/AmpScale` | float | Amplitude scale factor k | +| `/Tide/Local/MatchCount` | int | Number of matched observed/predicted events | +| `/Tide/Local/Status` | int | `0` no data, `1` time only, `2` time + amplitude | + +### Tide Event Slots + +Eight event slots provide the next and previous two highs and two lows. +Each slot has four sub-paths: + +| Slot | Sub-paths | +|---|---| +| `NextHigh1`, `NextHigh2` | `/Time`, `/Depth`, `/ObsTime`, `/ObsDepth` | +| `NextLow1`, `NextLow2` | `/Time`, `/Depth`, `/ObsTime`, `/ObsDepth` | +| `PrevHigh1`, `PrevHigh2` | `/Time`, `/Depth`, `/ObsTime`, `/ObsDepth` | +| `PrevLow1`, `PrevLow2` | `/Time`, `/Depth`, `/ObsTime`, `/ObsDepth` | + +Full path example: `/Tide/NextHigh1/Time` (Unix timestamp), +`/Tide/NextHigh1/Depth` (meters). + +`ObsTime` and `ObsDepth` are populated when a detected observed event has +been matched to this predicted slot; otherwise they are `null`. + +### Model Metadata + +| Path | Type | Description | +|---|---|---| +| `/Tide/LastModelRun` | int | Unix timestamp of the last prediction run | +| `/Tide/DataSource` | string | Source identifier, e.g. `noaa:8722670`, `grid` | +| `/Tide/ChartDepth` | float | Calibrated chart depth offset in meters | +| `/Tide/DatumOffset` | float | MSL above MLLW in meters | +| `/Tide/ModelLocation/Lat` | float | Latitude used for model | +| `/Tide/ModelLocation/Lon` | float | Longitude used for model | + +### Station Info + +| Path | Type | Description | +|---|---|---| +| `/Tide/Station/Id` | string | NOAA station ID | +| `/Tide/Station/Name` | string | Station name | +| `/Tide/Station/Distance` | float | Distance in km | +| `/Tide/Station/Lat` | float | Station latitude | +| `/Tide/Station/Lon` | float | Station longitude | +| `/Tide/Station/Type` | string | `R` (reference/harmonic) or `S` (subordinate) | +| `/Tide/Station/RefId` | string | Reference station ID (subordinate stations only) | +| `/Tide/NearbyStations` | string | JSON array of nearby stations for UI selection | + +### Settings (writable) + +| Path | Type | Default | Description | +|---|---|---|---| +| `/Settings/Enabled` | int | `1` | `0` disables the service | +| `/Settings/MinTideAmplitude` | float | `0.3` | Minimum amplitude for observed tide detection | +| `/Settings/Units` | int | `0` | Display unit preference | +| `/Settings/DatumOffset` | float | `-1.0` | Manual datum offset in meters (`-1` = automatic) | +| `/Settings/StationOverride` | string | `""` | Manual NOAA station ID (empty = automatic selection) | + +## Installation + +The service is designed for Venus OS (Victron Energy). It installs to +`/data/dbus-tides/` and registers itself with the daemontools service +supervisor. + +```sh +# Install +chmod +x install.sh +./install.sh + +# Uninstall (preserves data in /data/dbus-tides/) +./uninstall.sh +``` + +The install script: +- Locates `velib_python` and creates a symlink +- Links the service directory into `/service/` (or `/opt/victronenergy/service/`) +- Creates `/var/log/dbus-tides/` for multilog output +- Adds an `rc.local` entry so the service persists across firmware updates + +## Building + +```sh +./build-package.sh # dbus-tides-1.0.19.tar.gz +./build-package.sh --version 1.1.0 # override version +./build-package.sh --output /tmp/release # custom output dir +``` + +Produces a `.tar.gz` package and a `.sha256` checksum file. + +## Development Tools + +Tools in the `tools/` directory require the dependencies listed in +`tools/requirements-dev.txt` (`pyTMD`, `numpy`, `scipy`, `pyproj`, +`h5netcdf`, `xarray`). + +| Tool | Purpose | +|---|---| +| `build_noaa_stations.py` | Fetch NOAA harmonic and subordinate station data from the CO-OPS API | +| `build_coastal_grid.py` | Build a 0.25° coastal constituent grid from pyTMD / GOT4.10c | +| `predict_tides.py` | Standalone tide prediction for a given lat/lon | +| `extract_constituents.py` | Extract constituents for a single position from pyTMD | + +## Configuration + +All tunable parameters are in `config.py`. Key settings: + +| Parameter | Default | Description | +|---|---|---| +| `DEPTH_SAMPLE_INTERVAL` | 10 s | Depth sensor polling rate | +| `DEPTH_AVG_WINDOW` | 300 s | Averaging window for raw readings | +| `DEPTH_HISTORY_HOURS` | 96 h | History retention | +| `MIN_TIDE_AMPLITUDE` | 0.4 m | Minimum observed tide range | +| `SPEED_THRESHOLD_MS` | 1.03 m/s | Speed gate (≈ 2 knots) | +| `STATIONARY_RADIUS_METERS` | 9260 m | Anchorage radius (5 nm) | +| `CHART_DEPTH_RECAL_METERS` | 152 m | Movement to trigger recalibration | +| `MODEL_RERUN_RADIUS_METERS` | 37040 m | Movement to re-run model (20 nm) | +| `PREDICTION_HOURS` | 48 h | Prediction horizon | +| `PREDICTION_INTERVAL` | 900 s | Prediction time step (15 min) | +| `NOAA_STATION_MAX_DISTANCE` | 92600 m | Max station distance (50 nm) | +| `BESTFIT_MIN_HISTORY_HOURS` | 12 h | History required for best-fit scoring | +| `ADAPTATION_MAX_MATCH_WINDOW` | 10800 s | Max time difference for event matching | +| `RESIDUAL_DECAY_HOURS` | 6 h | Residual correction decay half-life | + +## License + +See the project repository for license details. diff --git a/dbus-tides/build-package.sh b/dbus-tides/build-package.sh new file mode 100755 index 0000000..9560a1a --- /dev/null +++ b/dbus-tides/build-package.sh @@ -0,0 +1,189 @@ +#!/bin/bash +# +# Build script for dbus-tides Venus OS package +# +# Creates a tar.gz package that can be: +# 1. Copied to a Venus OS device (Cerbo GX, Venus GX, etc.) +# 2. Untarred to /data/ +# 3. Installed by running install.sh +# +# Usage: +# ./build-package.sh # Creates package with default name +# ./build-package.sh --version 1.0.0 # Creates package with version in name +# ./build-package.sh --output /path/ # Specify output directory +# +# Installation on Venus OS: +# scp dbus-tides-*.tar.gz root@:/data/ +# ssh root@ +# cd /data && tar -xzf dbus-tides-*.tar.gz +# bash /data/dbus-tides/install.sh +# + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +VERSION=$(grep -oP "^VERSION\s*=\s*['\"]\\K[^'\"]*" "$SCRIPT_DIR/config.py" 2>/dev/null || echo "0.0.0") +OUTPUT_DIR="$SCRIPT_DIR" +PACKAGE_NAME="dbus-tides" + +while [[ $# -gt 0 ]]; do + case $1 in + --version|-v) + VERSION="$2" + shift 2 + ;; + --output|-o) + OUTPUT_DIR="$2" + shift 2 + ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " -v, --version VERSION Set package version (default: from config.py)" + echo " -o, --output PATH Output directory (default: script directory)" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +BUILD_DATE=$(date -u +"%Y-%m-%d %H:%M:%S UTC") +BUILD_TIMESTAMP=$(date +%Y%m%d%H%M%S) + +BUILD_DIR=$(mktemp -d) +PACKAGE_DIR="$BUILD_DIR/$PACKAGE_NAME" + +echo "==================================================" +echo "Building $PACKAGE_NAME package" +echo "==================================================" +echo "Version: $VERSION" +echo "Build date: $BUILD_DATE" +echo "Source: $SCRIPT_DIR" +echo "Output: $OUTPUT_DIR" +echo "" + +echo "1. Creating package structure..." +mkdir -p "$PACKAGE_DIR" +mkdir -p "$PACKAGE_DIR/service/log" +mkdir -p "$PACKAGE_DIR/constituents" + +[ "$(uname)" = "Darwin" ] && export COPYFILE_DISABLE=1 + +echo "2. Copying application files..." +cp "$SCRIPT_DIR/tides.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/config.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/depth_recorder.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/tide_detector.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/tide_harmonics.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/tide_predictor.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/tide_merger.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/tide_adapter.py" "$PACKAGE_DIR/" + +echo "3. Copying constituent data..." +if [ -f "$SCRIPT_DIR/constituents/noaa_stations.json.gz" ]; then + cp "$SCRIPT_DIR/constituents/noaa_stations.json.gz" "$PACKAGE_DIR/constituents/" + echo " Included noaa_stations.json.gz" +else + echo " NOTE: constituents/noaa_stations.json.gz not found." + echo " Run tools/build_noaa_stations.py on a dev machine to include NOAA station data." +fi +if [ -f "$SCRIPT_DIR/constituents/coastal_grid.json.gz" ]; then + cp "$SCRIPT_DIR/constituents/coastal_grid.json.gz" "$PACKAGE_DIR/constituents/" + echo " Included coastal_grid.json.gz" +else + echo " WARNING: constituents/coastal_grid.json.gz not found." + echo " Run tools/build_coastal_grid.py on a dev machine first, or the" + echo " service will rely on manual JSON at runtime." +fi +if ls "$SCRIPT_DIR"/constituents/*.json >/dev/null 2>&1; then + cp "$SCRIPT_DIR"/constituents/*.json "$PACKAGE_DIR/constituents/" + echo " Included custom constituent JSON files" +fi + +echo "4. Copying service files..." +cp "$SCRIPT_DIR/service/run" "$PACKAGE_DIR/service/" +cp "$SCRIPT_DIR/service/log/run" "$PACKAGE_DIR/service/log/" + +echo "5. Copying installation scripts..." +cp "$SCRIPT_DIR/install.sh" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/uninstall.sh" "$PACKAGE_DIR/" + +echo "6. Copying documentation..." +if [ -f "$SCRIPT_DIR/README.md" ]; then + cp "$SCRIPT_DIR/README.md" "$PACKAGE_DIR/" +fi + +echo "7. Creating version info..." +cat > "$PACKAGE_DIR/VERSION" << EOF +Package: $PACKAGE_NAME +Version: $VERSION +Build Date: $BUILD_DATE +Build Timestamp: $BUILD_TIMESTAMP + +Installation: + 1. Copy to Venus OS: scp $PACKAGE_NAME-$VERSION.tar.gz root@:/data/ + 2. SSH to device: ssh root@ + 3. Extract: cd /data && tar -xzf $PACKAGE_NAME-$VERSION.tar.gz + 4. Install: bash /data/$PACKAGE_NAME/install.sh +EOF + +echo "8. Setting permissions..." +chmod +x "$PACKAGE_DIR/tides.py" +chmod +x "$PACKAGE_DIR/install.sh" +chmod +x "$PACKAGE_DIR/uninstall.sh" +chmod +x "$PACKAGE_DIR/service/run" +chmod +x "$PACKAGE_DIR/service/log/run" + +mkdir -p "$OUTPUT_DIR" + +TARBALL_NAME="$PACKAGE_NAME-$VERSION.tar.gz" +OUTPUT_DIR_ABS="$(cd "$OUTPUT_DIR" && pwd)" +TARBALL_PATH="$OUTPUT_DIR_ABS/$TARBALL_NAME" + +echo "9. Creating package archive..." +cd "$BUILD_DIR" +if [ "$(uname)" = "Darwin" ]; then + if command -v xattr >/dev/null 2>&1; then + xattr -cr "$PACKAGE_NAME" + fi +fi +tar --format=ustar -czf "$TARBALL_PATH" "$PACKAGE_NAME" + +if command -v sha256sum >/dev/null 2>&1; then + CHECKSUM=$(sha256sum "$TARBALL_PATH" | cut -d' ' -f1) +else + CHECKSUM=$(shasum -a 256 "$TARBALL_PATH" | cut -d' ' -f1) +fi +echo "$CHECKSUM $TARBALL_NAME" > "$OUTPUT_DIR_ABS/$TARBALL_NAME.sha256" + +echo "10. Cleaning up..." +rm -rf "$BUILD_DIR" + +if [ "$(uname)" = "Darwin" ]; then + FILE_SIZE=$(du -h "$TARBALL_PATH" | cut -f1) +else + FILE_SIZE=$(du -h "$TARBALL_PATH" | cut -f1) +fi + +echo "" +echo "==================================================" +echo "Build complete!" +echo "==================================================" +echo "" +echo "Package: $TARBALL_PATH" +echo "Size: $FILE_SIZE" +echo "SHA256: $CHECKSUM" +echo "" +echo "Installation on Venus OS:" +echo " scp $TARBALL_PATH root@:/data/" +echo " ssh root@" +echo " cd /data" +echo " tar -xzf $TARBALL_NAME" +echo " bash /data/$PACKAGE_NAME/install.sh" +echo "" diff --git a/dbus-tides/config.py b/dbus-tides/config.py new file mode 100644 index 0000000..d01e669 --- /dev/null +++ b/dbus-tides/config.py @@ -0,0 +1,156 @@ +""" +Configuration for dbus-tides Venus OS service. + +All tunable parameters in one place for easy adjustment. +""" + +# ============================================================================= +# D-BUS SERVICE CONFIGURATION +# ============================================================================= + +SERVICE_NAME = 'com.victronenergy.tides' + +# ============================================================================= +# DEPTH SAMPLING +# ============================================================================= + +# How often to read the depth sensor (seconds) +DEPTH_SAMPLE_INTERVAL = 10 + +# Window for averaging raw depth readings to smooth wave noise (seconds) +DEPTH_AVG_WINDOW = 300 # 5 minutes + +# How many hours of 5-min averages to keep in the ring buffer and DB +DEPTH_HISTORY_HOURS = 96 + +# Median filter kernel size (number of 5-min samples). Odd integer. +# Removes impulse noise from anchor swing without shifting timing. +MEDIAN_FILTER_KERNEL = 5 + +# Window for smoothing the 5-min averages for trend/slope detection (seconds). +# Triangular-weighted moving average; wider than the raw averaging window to +# suppress anchor-swing residuals that survive the 5-min mean. +SMOOTHING_WINDOW = 3600 # 60 minutes + +# ============================================================================= +# TIDE DETECTION +# ============================================================================= + +# Minimum depth change (meters) between consecutive high/low events. +# Must exceed the peak-to-peak residual noise in the smoothed curve. +MIN_TIDE_AMPLITUDE = 0.4 + +# Depth excursion (meters) required after a candidate extremum before it is +# reported. In calm conditions the smoothed curve moves away quickly; in +# noisy conditions (anchor swing) confirmation is naturally delayed until +# enough data accumulates to fit an unambiguous trend. +TIDE_CONFIRMATION_DEPTH = 0.2 + +# ============================================================================= +# SPEED GATE +# ============================================================================= + +# Maximum vessel speed (m/s) to accept depth readings. +# 2 knots = 1.02889 m/s +SPEED_THRESHOLD_MS = 1.03 + +# ============================================================================= +# LOCATION THRESHOLDS +# ============================================================================= + +# How often to sample GPS for position tracking (seconds) +GPS_SAMPLE_INTERVAL = 60 + +# GPS history buffer size for movement detection +GPS_HISTORY_SIZE = 4 + +# GPS history sample interval (seconds) +GPS_HISTORY_SAMPLE_INTERVAL = 900 # 15 minutes + +# Within this radius (meters), tidal timing is considered consistent. +# Depth observations remain valid for the same anchorage. +# 5 nautical miles = 9260 meters +STATIONARY_RADIUS_METERS = 9260 + +# When the vessel moves more than this distance (meters), recalibrate +# chart_depth using only recent observations from the new position. +# Tidal correction factors (Dt, k) are preserved since the tidal +# characteristics are unchanged within the stationary radius. +# 500 feet = 152.4 meters +CHART_DEPTH_RECAL_METERS = 152.4 + +# Beyond this radius (meters), re-interpolate tidal constituents from grid. +# 20 nautical miles = 37040 meters +MODEL_RERUN_RADIUS_METERS = 37040 + +# Maximum time between tide predictions (seconds) even if stationary +MODEL_RERUN_INTERVAL = 86400 # 1 day + +# ============================================================================= +# LOCAL ADAPTATION +# ============================================================================= + +# Maximum time difference (seconds) to match an observed event to a predicted +# extremum. 3 hours accommodates large coastal propagation delays. +ADAPTATION_MAX_MATCH_WINDOW = 10800 + +# Minimum number of matched observed/predicted pairs before applying a +# timing correction. +ADAPTATION_MIN_MATCHES = 2 + +# Minimum number of high+low range pairs before computing amplitude scaling. +ADAPTATION_MIN_RANGE_PAIRS = 1 + +# Smoothing kernel width (seconds) for the observation-vs-prediction residual. +# Must be wide enough to filter 5-min noise but narrow enough to preserve +# tidal shape (~1.5 hours). +RESIDUAL_SMOOTHING_WINDOW = 5400 # 1.5 hours + +# Exponential decay half-life (hours) for extrapolating the residual +# correction beyond the last observation into the future. +RESIDUAL_DECAY_HOURS = 6 + +# ============================================================================= +# TIDE PREDICTION +# ============================================================================= + +# How many hours ahead to predict +PREDICTION_HOURS = 48 + +# Time step for the predicted tide curve (seconds) +PREDICTION_INTERVAL = 900 # 15-minute intervals + +# ============================================================================= +# PATH CONFIGURATION +# ============================================================================= + +DATA_DIR = '/data/dbus-tides' +DB_FILE = DATA_DIR + '/depth_history.db' +CONSTITUENTS_DIR = DATA_DIR + '/constituents' +GRID_FILE_NAME = 'coastal_grid.json.gz' +NOAA_STATIONS_FILE_NAME = 'noaa_stations.json.gz' + +# Maximum distance (meters) to a NOAA station for it to be used. +# Observation-derived constituents are preferred over the global model +# when the vessel is within this radius of a station. +# 50 nautical miles = 92600 meters +NOAA_STATION_MAX_DISTANCE = 92600 + +# Number of nearby NOAA stations to report for UI selection +NOAA_NEARBY_COUNT = 5 + +# Minimum hours of depth history required before attempting +# best-fit station scoring against observed data +BESTFIT_MIN_HISTORY_HOURS = 12 + +# ============================================================================= +# LOGGING CONFIGURATION +# ============================================================================= + +LOGGING_CONFIG = { + 'level': 'INFO', + 'console': True, + 'include_timestamp': False, +} + +VERSION = '1.0.19' diff --git a/dbus-tides/constituents/coastal_grid.json.gz b/dbus-tides/constituents/coastal_grid.json.gz new file mode 100644 index 0000000..59bbb81 Binary files /dev/null and b/dbus-tides/constituents/coastal_grid.json.gz differ diff --git a/dbus-tides/constituents/noaa_stations.json.gz b/dbus-tides/constituents/noaa_stations.json.gz new file mode 100644 index 0000000..8f86814 Binary files /dev/null and b/dbus-tides/constituents/noaa_stations.json.gz differ diff --git a/dbus-tides/depth_recorder.py b/dbus-tides/depth_recorder.py new file mode 100644 index 0000000..bdc7007 --- /dev/null +++ b/dbus-tides/depth_recorder.py @@ -0,0 +1,211 @@ +""" +Depth recording with averaging and persistence. + +Collects raw depth readings, computes 5-minute averages to smooth wave noise, +stores them in a 96-hour circular buffer, and persists to SQLite across restarts. + +Each record includes the vessel's GPS position so that observations can be +filtered to only those taken near the current anchorage. +""" + +import logging +import math +import os +import sqlite3 +import time +from collections import deque + +from config import ( + DEPTH_AVG_WINDOW, + DEPTH_HISTORY_HOURS, + DB_FILE, + STATIONARY_RADIUS_METERS, +) + +logger = logging.getLogger('DepthRecorder') + +_SLOTS = DEPTH_HISTORY_HOURS * 3600 // DEPTH_AVG_WINDOW + + +def _haversine(lat1, lon1, lat2, lon2): + """Great-circle distance in meters between two GPS coordinates.""" + R = 6371000 + phi1 = math.radians(lat1) + phi2 = math.radians(lat2) + dphi = math.radians(lat2 - lat1) + dlam = math.radians(lon2 - lon1) + a = (math.sin(dphi / 2) ** 2 + + math.cos(phi1) * math.cos(phi2) * math.sin(dlam / 2) ** 2) + return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) + + +class DepthRecorder: + """Accumulates raw depth readings, averages them, and stores history.""" + + def __init__(self, db_path=None): + self.db_path = db_path or DB_FILE + + self._raw_window = [] + self._window_start = 0.0 + + # Ring buffer: (timestamp, depth_avg, lat, lon) + self.history = deque(maxlen=_SLOTS) + + self._init_db() + self._load_history() + + # ------------------------------------------------------------------ + # Database + # ------------------------------------------------------------------ + + def _init_db(self): + db_dir = os.path.dirname(self.db_path) + if db_dir and not os.path.exists(db_dir): + try: + os.makedirs(db_dir, exist_ok=True) + except OSError: + pass + try: + self._conn = sqlite3.connect(self.db_path) + self._conn.execute( + 'CREATE TABLE IF NOT EXISTS depth_history ' + '(timestamp REAL PRIMARY KEY, depth_avg REAL, lat REAL, lon REAL)') + self._conn.execute( + 'CREATE INDEX IF NOT EXISTS idx_ts ON depth_history(timestamp)') + self._conn.commit() + logger.info(f"Database opened: {self.db_path}") + except sqlite3.Error as e: + logger.error(f"Database init failed: {e}") + self._conn = None + + def _load_history(self): + """Load recent history from SQLite on startup.""" + if not self._conn: + return + try: + cutoff = time.time() - DEPTH_HISTORY_HOURS * 3600 + rows = self._conn.execute( + 'SELECT timestamp, depth_avg, lat, lon FROM depth_history ' + 'WHERE timestamp > ? ORDER BY timestamp', (cutoff,) + ).fetchall() + for ts, depth, lat, lon in rows: + self.history.append((ts, depth, lat, lon)) + logger.info(f"Loaded {len(rows)} depth history records") + except sqlite3.Error as e: + logger.warning(f"Failed to load history: {e}") + + def _persist(self, ts, depth_avg, lat, lon): + if not self._conn: + return + try: + self._conn.execute( + 'INSERT OR REPLACE INTO depth_history ' + '(timestamp, depth_avg, lat, lon) VALUES (?, ?, ?, ?)', + (ts, depth_avg, lat, lon)) + self._conn.commit() + except sqlite3.Error as e: + logger.warning(f"Failed to persist depth: {e}") + + def _prune_db(self): + """Remove records older than the history window.""" + if not self._conn: + return + try: + cutoff = time.time() - DEPTH_HISTORY_HOURS * 3600 + self._conn.execute( + 'DELETE FROM depth_history WHERE timestamp < ?', (cutoff,)) + self._conn.commit() + except sqlite3.Error: + pass + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def add_reading(self, depth, lat=None, lon=None, now=None): + """Add a raw depth reading. Returns a new average if the window closed.""" + if now is None: + now = time.time() + + if not self._raw_window: + self._window_start = now + + self._raw_window.append((now, depth, lat, lon)) + + elapsed = now - self._window_start + if elapsed >= DEPTH_AVG_WINDOW and self._raw_window: + return self._flush_window() + + return None + + def _flush_window(self): + """Compute the average of the current window and store it.""" + if not self._raw_window: + return None + + depths = [d for _, d, _, _ in self._raw_window] + avg_depth = sum(depths) / len(depths) + mid_ts = (self._window_start + self._raw_window[-1][0]) / 2.0 + + last_lat = None + last_lon = None + for _, _, lat, lon in reversed(self._raw_window): + if lat is not None and lon is not None: + last_lat, last_lon = lat, lon + break + + self.history.append((mid_ts, avg_depth, last_lat, last_lon)) + self._persist(mid_ts, avg_depth, last_lat, last_lon) + + self._raw_window.clear() + self._window_start = 0.0 + + if len(self.history) % 100 == 0: + self._prune_db() + + return (mid_ts, avg_depth) + + def get_history(self, lat=None, lon=None, radius=None): + """Return list of (timestamp, depth) tuples, optionally filtered by position. + + Args: + lat, lon: current vessel position for proximity filtering + radius: max distance in meters (default: STATIONARY_RADIUS_METERS) + + When lat/lon are provided, only records taken within `radius` of + that position are returned. Records with no stored position are + included (they predate GPS-tagged recording). + """ + if radius is None: + radius = STATIONARY_RADIUS_METERS + + result = [] + for entry in self.history: + ts, depth = entry[0], entry[1] + rec_lat = entry[2] if len(entry) > 2 else None + rec_lon = entry[3] if len(entry) > 3 else None + + if lat is not None and lon is not None: + if (rec_lat is not None and rec_lon is not None + and _haversine(lat, lon, rec_lat, rec_lon) > radius): + continue + + result.append((ts, depth)) + + return result + + def get_latest_depth(self): + """Return the most recent averaged depth, or None.""" + if self.history: + return self.history[-1][1] + if self._raw_window: + depths = [d for _, d, _, _ in self._raw_window] + return sum(depths) / len(depths) + return None + + def close(self): + if self._conn: + try: + self._conn.close() + except sqlite3.Error: + pass diff --git a/dbus-tides/install.sh b/dbus-tides/install.sh new file mode 100644 index 0000000..f708b76 --- /dev/null +++ b/dbus-tides/install.sh @@ -0,0 +1,170 @@ +#!/bin/bash +# +# Installation script for dbus-tides on Venus OS +# +# Run this on the Venus OS device after copying files to /data/dbus-tides/ +# +# Usage: +# chmod +x install.sh +# ./install.sh +# + +set -e + +INSTALL_DIR="/data/dbus-tides" + +# Find velib_python +VELIB_DIR="" +if [ -d "/opt/victronenergy/velib_python" ]; then + VELIB_DIR="/opt/victronenergy/velib_python" +else + for candidate in \ + "/opt/victronenergy/dbus-systemcalc-py/ext/velib_python" \ + "/opt/victronenergy/dbus-generator/ext/velib_python" \ + "/opt/victronenergy/dbus-mqtt/ext/velib_python" \ + "/opt/victronenergy/dbus-digitalinputs/ext/velib_python" \ + "/opt/victronenergy/vrmlogger/ext/velib_python" + do + if [ -d "$candidate" ] && [ -f "$candidate/vedbus.py" ]; then + VELIB_DIR="$candidate" + break + fi + done +fi + +if [ -z "$VELIB_DIR" ]; then + VEDBUS_PATH=$(find /opt/victronenergy -name "vedbus.py" -path "*/velib_python/*" 2>/dev/null | head -1) + if [ -n "$VEDBUS_PATH" ]; then + VELIB_DIR=$(dirname "$VEDBUS_PATH") + fi +fi + +# Determine service directory +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +elif [ -L "/service" ]; then + SERVICE_DIR=$(readlink -f /service) +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "==================================================" +echo "dbus-tides - Installation" +echo "==================================================" + +if [ ! -d "$SERVICE_DIR" ]; then + echo "ERROR: This doesn't appear to be a Venus OS device." + echo " Service directory not found." + exit 1 +fi + +echo "Detected service directory: $SERVICE_DIR" + +if [ ! -f "$INSTALL_DIR/tides.py" ]; then + echo "ERROR: Installation files not found in $INSTALL_DIR" + echo " Please copy all files to $INSTALL_DIR first." + exit 1 +fi +if [ ! -f "$INSTALL_DIR/service/run" ]; then + echo "ERROR: service/run not found. The package is incomplete." + exit 1 +fi + +echo "1. Making scripts executable..." +chmod +x "$INSTALL_DIR/service/run" +chmod +x "$INSTALL_DIR/service/log/run" +chmod +x "$INSTALL_DIR/tides.py" + +echo "2. Creating velib_python symlink..." +if [ -z "$VELIB_DIR" ]; then + echo "ERROR: Could not find velib_python on this system." + exit 1 +fi +echo " Found velib_python at: $VELIB_DIR" +mkdir -p "$INSTALL_DIR/ext" +if [ -L "$INSTALL_DIR/ext/velib_python" ]; then + CURRENT_TARGET=$(readlink "$INSTALL_DIR/ext/velib_python") + if [ "$CURRENT_TARGET" != "$VELIB_DIR" ]; then + echo " Updating symlink (was: $CURRENT_TARGET)" + rm "$INSTALL_DIR/ext/velib_python" + fi +fi +if [ ! -L "$INSTALL_DIR/ext/velib_python" ]; then + ln -s "$VELIB_DIR" "$INSTALL_DIR/ext/velib_python" + echo " Symlink created: $INSTALL_DIR/ext/velib_python -> $VELIB_DIR" +else + echo " Symlink already exists" +fi + +echo "3. Creating data directories..." +mkdir -p "$INSTALL_DIR/constituents" + +echo "4. Creating service symlink..." +if [ -L "$SERVICE_DIR/dbus-tides" ]; then + echo " Service link already exists, removing old link..." + rm "$SERVICE_DIR/dbus-tides" +fi +if [ -e "$SERVICE_DIR/dbus-tides" ]; then + rm -rf "$SERVICE_DIR/dbus-tides" +fi +ln -s "$INSTALL_DIR/service" "$SERVICE_DIR/dbus-tides" + +if [ -L "$SERVICE_DIR/dbus-tides" ]; then + echo " Symlink created: $SERVICE_DIR/dbus-tides -> $INSTALL_DIR/service" +else + echo "ERROR: Failed to create service symlink" + exit 1 +fi + +echo "5. Creating log directory..." +mkdir -p /var/log/dbus-tides + +echo "6. Setting up rc.local for persistence..." +RC_LOCAL="/data/rc.local" +if [ ! -f "$RC_LOCAL" ]; then + echo "#!/bin/bash" > "$RC_LOCAL" + chmod +x "$RC_LOCAL" +fi + +if ! grep -q "dbus-tides" "$RC_LOCAL"; then + echo "" >> "$RC_LOCAL" + echo "# dbus-tides" >> "$RC_LOCAL" + echo "if [ ! -L $SERVICE_DIR/dbus-tides ]; then" >> "$RC_LOCAL" + echo " ln -s /data/dbus-tides/service $SERVICE_DIR/dbus-tides" >> "$RC_LOCAL" + echo "fi" >> "$RC_LOCAL" + echo " Added to rc.local for persistence across firmware updates" +else + echo " Already in rc.local" +fi + +echo "7. Activating service..." +sleep 2 +if command -v svstat >/dev/null 2>&1; then + if svstat "$SERVICE_DIR/dbus-tides" 2>/dev/null | grep -q "up"; then + echo " Service is running" + else + echo " Waiting for service to start..." + sleep 3 + fi +fi + +echo "" +echo "==================================================" +echo "Installation complete!" +echo "==================================================" +echo "" + +if command -v svstat >/dev/null 2>&1; then + echo "Current status:" + svstat "$SERVICE_DIR/dbus-tides" 2>/dev/null || echo " Service not yet detected by svscan" + echo "" +fi + +echo "To check status:" +echo " svstat $SERVICE_DIR/dbus-tides" +echo "" +echo "To view logs:" +echo " tail -F /var/log/dbus-tides/current | tai64nlocal" +echo "" diff --git a/dbus-tides/service/log/run b/dbus-tides/service/log/run new file mode 100644 index 0000000..213fc99 --- /dev/null +++ b/dbus-tides/service/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec multilog t s99999 n8 /var/log/dbus-tides diff --git a/dbus-tides/service/run b/dbus-tides/service/run new file mode 100644 index 0000000..aa4bd44 --- /dev/null +++ b/dbus-tides/service/run @@ -0,0 +1,5 @@ +#!/bin/sh +exec 2>&1 +cd /data/dbus-tides +export PYTHONPATH="/data/dbus-tides/ext/velib_python:$PYTHONPATH" +exec python3 /data/dbus-tides/tides.py diff --git a/dbus-tides/tide_adapter.py b/dbus-tides/tide_adapter.py new file mode 100644 index 0000000..97d09a7 --- /dev/null +++ b/dbus-tides/tide_adapter.py @@ -0,0 +1,360 @@ +""" +Tide adapter -- computes timing offset and amplitude scaling from +observed tide events matched against harmonic model predictions. + +Local geography causes tides to arrive earlier/later and with +different amplitude compared to the open-ocean model. The adapter +estimates two correction factors: + + Dt = timing offset (seconds, + means local tide arrives later) + k = amplitude scale factor (1.0 = no scaling) + +The corrected prediction is: + + local_height(t) = k * predicted_height(t - Dt) +""" + +import logging +import math + +from config import ( + ADAPTATION_MAX_MATCH_WINDOW, + ADAPTATION_MIN_MATCHES, + ADAPTATION_MIN_RANGE_PAIRS, + RESIDUAL_DECAY_HOURS, + RESIDUAL_SMOOTHING_WINDOW, +) + +logger = logging.getLogger('TideAdapter') + + +class TideAdapter: + """Matches observed tide events to predictions and computes corrections.""" + + def __init__(self): + self._time_offset = 0.0 + self._amp_scale = 1.0 + self._match_count = 0 + self._status = 0 # 0=no data, 1=time only, 2=time+amplitude + + @property + def time_offset(self): + return self._time_offset + + @property + def amp_scale(self): + return self._amp_scale + + @property + def match_count(self): + return self._match_count + + @property + def status(self): + return self._status + + def reset(self): + """Reset corrections (e.g. when vessel moves).""" + self._time_offset = 0.0 + self._amp_scale = 1.0 + self._match_count = 0 + self._status = 0 + logger.info("Adaptation reset") + + def update(self, observed_events, predicted_extrema): + """Match observed events to predicted extrema and compute corrections. + + Args: + observed_events: list of (timestamp, depth, "high"|"low") + from TideDetector + predicted_extrema: list of (timestamp, height, "high"|"low") + from tide_harmonics.find_extrema() + + Returns: + True if corrections were updated. + """ + if not observed_events or not predicted_extrema: + return False + + matches = self._match_events(observed_events, predicted_extrema) + if len(matches) < ADAPTATION_MIN_MATCHES: + return False + + time_offsets = [obs_ts - pred_ts + for obs_ts, _, pred_ts, _, _ in matches] + self._time_offset = _median(time_offsets) + self._match_count = len(matches) + + highs_obs = [(obs_ts, obs_d) for obs_ts, obs_d, _, _, t in matches + if t == "high"] + lows_obs = [(obs_ts, obs_d) for obs_ts, obs_d, _, _, t in matches + if t == "low"] + highs_pred = [(pred_ts, pred_h) for _, _, pred_ts, pred_h, t in matches + if t == "high"] + lows_pred = [(pred_ts, pred_h) for _, _, pred_ts, pred_h, t in matches + if t == "low"] + + amp_ratios = self._compute_amplitude_ratios( + highs_obs, lows_obs, highs_pred, lows_pred) + + if len(amp_ratios) >= ADAPTATION_MIN_RANGE_PAIRS: + self._amp_scale = _median(amp_ratios) + self._amp_scale = max(0.3, min(self._amp_scale, 3.0)) + self._status = 2 + logger.info( + "Adaptation: Dt=%.0fs (%.1fmin), k=%.2f from %d matches " + "(%d range pairs)", + self._time_offset, self._time_offset / 60.0, + self._amp_scale, self._match_count, len(amp_ratios)) + else: + self._amp_scale = 1.0 + self._status = 1 + logger.info( + "Adaptation: Dt=%.0fs (%.1fmin), k=1.0 (no range pairs) " + "from %d matches", + self._time_offset, self._time_offset / 60.0, + self._match_count) + + return True + + def adapt_curve(self, timestamps, heights): + """Apply timing offset and amplitude scaling to a prediction curve. + + Args: + timestamps: list of Unix timestamps (original prediction times) + heights: list of MSL-relative heights + + Returns: + (adapted_timestamps, adapted_heights) -- same length as input. + Timestamps are shifted by +Dt, heights are scaled by k. + """ + if self._status == 0: + return timestamps, heights + + dt = self._time_offset + k = self._amp_scale + + adapted_ts = [t + dt for t in timestamps] + adapted_h = [k * h for h in heights] + return adapted_ts, adapted_h + + def adapt_extrema(self, extrema): + """Apply corrections to predicted extrema. + + Args: + extrema: list of (timestamp, height, "high"|"low") + + Returns: + list of (timestamp, height, type) with corrected values. + """ + if self._status == 0: + return extrema + + dt = self._time_offset + k = self._amp_scale + return [(ts + dt, k * h, t) for ts, h, t in extrema] + + # ------------------------------------------------------------------ + # Internal + # ------------------------------------------------------------------ + + def _match_events(self, observed, predicted): + """Match each observed event to the nearest predicted event of the + same type within the match window. + + Returns list of (obs_ts, obs_depth, pred_ts, pred_height, type). + """ + matches = [] + used_pred = set() + + for obs_ts, obs_depth, obs_type in observed: + best_dist = ADAPTATION_MAX_MATCH_WINDOW + best_idx = -1 + + for i, (pred_ts, pred_h, pred_type) in enumerate(predicted): + if pred_type != obs_type: + continue + if i in used_pred: + continue + dist = abs(obs_ts - pred_ts) + if dist < best_dist: + best_dist = dist + best_idx = i + + if best_idx >= 0: + pred_ts, pred_h, pred_type = predicted[best_idx] + matches.append( + (obs_ts, obs_depth, pred_ts, pred_h, obs_type)) + used_pred.add(best_idx) + + return matches + + @staticmethod + def _compute_amplitude_ratios(highs_obs, lows_obs, highs_pred, lows_pred): + """Compute amplitude ratios from matched high/low pairs. + + For each consecutive (high, low) or (low, high) pair in the + observations, compute the ratio of observed range to predicted range. + """ + if not highs_obs or not lows_obs or not highs_pred or not lows_pred: + return [] + + obs_range = abs(highs_obs[0][1] - lows_obs[0][1]) + pred_range = abs(highs_pred[0][1] - lows_pred[0][1]) + + ratios = [] + if pred_range > 0.05: + ratios.append(obs_range / pred_range) + + for i in range(1, min(len(highs_obs), len(lows_obs), + len(highs_pred), len(lows_pred))): + or_i = abs(highs_obs[i][1] - lows_obs[i][1]) + pr_i = abs(highs_pred[i][1] - lows_pred[i][1]) + if pr_i > 0.05: + ratios.append(or_i / pr_i) + + return ratios + + # ------------------------------------------------------------------ + # Residual curve correction + # ------------------------------------------------------------------ + + def compute_residual_correction(self, obs_history, pred_at_func, + chart_depth): + """Compute a smoothed residual between observations and adapted model. + + Args: + obs_history: list of (timestamp, depth) from the depth recorder + pred_at_func: callable(timestamps) -> list of MSL-relative heights + chart_depth: static depth offset (meters) + + Returns: + list of (timestamp, smoothed_residual) or empty list. + """ + if not obs_history or len(obs_history) < 6: + return [] + if self._status == 0: + return [] + + dt = self._time_offset + k = self._amp_scale + + obs_ts_list = [ts for ts, _ in obs_history] + obs_depths = [d for _, d in obs_history] + + shifted_ts = [ts - dt for ts in obs_ts_list] + raw_heights = pred_at_func(shifted_ts) + if raw_heights is None or len(raw_heights) != len(obs_ts_list): + return [] + + residuals = [] + for i in range(len(obs_ts_list)): + adapted_depth = chart_depth + k * raw_heights[i] + residuals.append(obs_depths[i] - adapted_depth) + + smoothed = _smooth_triangular(obs_ts_list, residuals, + RESIDUAL_SMOOTHING_WINDOW) + + self._residual_correction = list(zip(obs_ts_list, smoothed)) + n = len(self._residual_correction) + if n > 0: + logger.info( + "Residual correction: %d points, range [%.3f, %.3f]m", + n, min(smoothed), max(smoothed)) + return self._residual_correction + + def apply_residual_correction(self, pred_ts, pred_depths, residual_curve): + """Apply the smoothed residual to a prediction curve. + + In the overlap region the interpolated residual is added directly. + Beyond the last observation the residual decays exponentially. + + Args: + pred_ts: prediction timestamps + pred_depths: prediction depths (adapted + chart_depth) + residual_curve: list of (timestamp, residual) from + compute_residual_correction() + + Returns: + list of corrected depths (same length as pred_ts). + """ + if not residual_curve or not pred_ts: + return pred_depths + + rc_ts = [t for t, _ in residual_curve] + rc_val = [v for _, v in residual_curve] + rc_start = rc_ts[0] + rc_end = rc_ts[-1] + last_r = rc_val[-1] + + tau = RESIDUAL_DECAY_HOURS * 3600.0 / math.log(2.0) + + corrected = [] + for i, t in enumerate(pred_ts): + if t < rc_start: + r = rc_val[0] + elif t <= rc_end: + r = _interp(t, rc_ts, rc_val) + else: + elapsed = t - rc_end + r = last_r * math.exp(-elapsed / tau) + corrected.append(pred_depths[i] + r) + + return corrected + + +def _interp(target, xs, ys): + """Linear interpolation of ys at target within sorted xs.""" + if target <= xs[0]: + return ys[0] + if target >= xs[-1]: + return ys[-1] + lo, hi = 0, len(xs) - 1 + while hi - lo > 1: + mid = (lo + hi) // 2 + if xs[mid] <= target: + lo = mid + else: + hi = mid + dx = xs[hi] - xs[lo] + if dx == 0: + return ys[lo] + frac = (target - xs[lo]) / dx + return ys[lo] + frac * (ys[hi] - ys[lo]) + + +def _smooth_triangular(timestamps, values, window): + """Triangular-kernel smoothing of (timestamps, values). + + Each output value is a weighted average of all input values within + +/- window/2, weighted linearly by proximity (1 at centre, 0 at edge). + """ + half = window / 2.0 + n = len(values) + result = [0.0] * n + + for i in range(n): + t_i = timestamps[i] + w_sum = 0.0 + v_sum = 0.0 + for j in range(n): + dist = abs(timestamps[j] - t_i) + if dist > half: + continue + w = 1.0 - dist / half + w_sum += w + v_sum += w * values[j] + result[i] = v_sum / w_sum if w_sum > 0 else values[i] + + return result + + +def _median(values): + """Return the median of a list of numbers.""" + if not values: + return 0.0 + s = sorted(values) + n = len(s) + if n % 2 == 1: + return s[n // 2] + return (s[n // 2 - 1] + s[n // 2]) / 2.0 diff --git a/dbus-tides/tide_detector.py b/dbus-tides/tide_detector.py new file mode 100644 index 0000000..5aa599b --- /dev/null +++ b/dbus-tides/tide_detector.py @@ -0,0 +1,245 @@ +""" +Tide event detector. + +Multi-stage filtering pipeline detects high/low tide events from noisy +depth observations taken at anchor. + +Pipeline: + 1. Median filter removes impulse noise from anchor swing + 2. Triangular-weighted moving average extracts the tidal trend + 3. Slope reversals identify candidate extrema + 4. Iterative merge enforces strict high-low alternation + 5. Amplitude gate rejects sub-tidal oscillations + 6. Confirmation delay holds back the most recent candidate until the + smoothed curve has moved away from it, adapting naturally to + conditions: fast in calm water, slower in noisy anchor-swing. +""" + +import logging +import time +from collections import deque + +from config import ( + DEPTH_AVG_WINDOW, + MEDIAN_FILTER_KERNEL, + MIN_TIDE_AMPLITUDE, + SMOOTHING_WINDOW, + TIDE_CONFIRMATION_DEPTH, +) + +logger = logging.getLogger('TideDetector') + +_SMOOTH_SLOTS = SMOOTHING_WINDOW // DEPTH_AVG_WINDOW + + +class TideDetector: + """Detects high and low tide events from smoothed depth observations.""" + + def __init__(self, min_amplitude=None): + self.min_amplitude = min_amplitude or MIN_TIDE_AMPLITUDE + self.events = deque(maxlen=50) + + def update(self, history): + """Process the full depth history and detect new tide events. + + Re-processes the entire smoothed history on each call. The most + recent candidate is held back until confirmed by a sustained depth + excursion, so in calm conditions events are reported promptly while + in noisy conditions (anchor swing) reporting waits until the trend + is unambiguous. + + Args: + history: list of (timestamp, depth_avg) from DepthRecorder + + Returns: + Newly detected event tuple (timestamp, depth, type) or None. + """ + if len(history) < _SMOOTH_SLOTS + 1: + return None + + smoothed = self._smooth(history) + if len(smoothed) < 2: + return None + + confirmed = self._detect_events(smoothed) + + new_event = None + for evt in confirmed: + if not self._already_reported(evt[0]): + self.events.append(evt) + logger.info( + "Detected %s tide: depth=%.2fm at %s", + evt[2], evt[1], + time.strftime('%H:%M', time.localtime(evt[0]))) + new_event = evt + + return new_event + + def _already_reported(self, ts): + """True if an event near this timestamp is already in the deque.""" + for existing in self.events: + if abs(existing[0] - ts) < SMOOTHING_WINDOW: + return True + return False + + # ------------------------------------------------------------------ + # Smoothing + # ------------------------------------------------------------------ + + def _smooth(self, history): + """Median pre-filter followed by triangular-weighted moving average.""" + if len(history) < _SMOOTH_SLOTS: + return list(history) + + filtered = self._median_filter(history) + return self._triangular_smooth(filtered) + + @staticmethod + def _median_filter(data): + """Remove impulse noise from anchor swing.""" + half = MEDIAN_FILTER_KERNEL // 2 + result = [] + for i in range(len(data)): + lo = max(0, i - half) + hi = min(len(data), i + half + 1) + depths = sorted(d for _, d in data[lo:hi]) + result.append((data[i][0], depths[len(depths) // 2])) + return result + + @staticmethod + def _triangular_smooth(data): + """Triangular-weighted moving average for clean frequency rolloff.""" + half = _SMOOTH_SLOTS // 2 + if len(data) < _SMOOTH_SLOTS: + return list(data) + + result = [] + for i in range(half, len(data) - half): + window = data[i - half:i + half + 1] + weights = [ + 1.0 - abs(j - half) / (half + 1) + for j in range(len(window)) + ] + w_sum = sum(weights) + avg_ts = sum( + w * ts for w, (ts, _) in zip(weights, window)) / w_sum + avg_d = sum( + w * d for w, (_, d) in zip(weights, window)) / w_sum + result.append((avg_ts, avg_d)) + return result + + # ------------------------------------------------------------------ + # Event detection + # ------------------------------------------------------------------ + + def _detect_events(self, smoothed): + """Find, merge, gate, and confirm tide events from smoothed data.""" + reversals = self._find_reversals(smoothed) + cleaned = self._merge_and_gate(reversals) + return self._confirm_events(smoothed, cleaned) + + @staticmethod + def _find_reversals(smoothed): + """Detect all slope sign changes in the smoothed series.""" + candidates = [] + prev_sign = 0 + for i in range(1, len(smoothed)): + ts_prev, d_prev = smoothed[i - 1] + ts_curr, d_curr = smoothed[i] + if d_curr > d_prev: + new_sign = 1 + elif d_curr < d_prev: + new_sign = -1 + else: + continue + if prev_sign != 0 and new_sign != prev_sign: + typ = "high" if prev_sign == 1 else "low" + candidates.append((ts_prev, d_prev, typ)) + prev_sign = new_sign + return candidates + + def _merge_and_gate(self, candidates): + """Iteratively merge same-type events and apply amplitude gate. + + When noise-induced mini-reversals produce consecutive same-type + candidates the most extreme depth is kept. Small-amplitude + oscillations are then pruned. Iteration continues until the + sequence is stable and strictly alternates high-low. + """ + events = list(candidates) + changed = True + while changed: + changed = False + + merged = [] + for ts, d, typ in events: + if merged and merged[-1][2] == typ: + prev_d = merged[-1][1] + if ((typ == "high" and d > prev_d) + or (typ == "low" and d < prev_d)): + merged[-1] = (ts, d, typ) + else: + merged.append((ts, d, typ)) + if len(merged) != len(events): + changed = True + + filtered = [merged[0]] if merged else [] + for i in range(1, len(merged)): + ts, d, typ = merged[i] + if abs(d - filtered[-1][1]) < self.min_amplitude: + changed = True + continue + filtered.append((ts, d, typ)) + events = filtered + return events + + @staticmethod + def _confirm_events(smoothed, events): + """Hold back the most recent event until confirmed. + + Confirmation requires smoothed depths after the candidate to have + moved away by at least TIDE_CONFIRMATION_DEPTH. The smoothing + window naturally makes this condition harder to meet in noisy + conditions, providing adaptive delay. + """ + if not events: + return events + + last_ts, last_d, last_type = events[-1] + max_excursion = 0.0 + for ts, d in smoothed: + if ts <= last_ts: + continue + if last_type == "high": + excursion = last_d - d + else: + excursion = d - last_d + if excursion > max_excursion: + max_excursion = excursion + + if max_excursion < TIDE_CONFIRMATION_DEPTH: + return events[:-1] + return events + + # ------------------------------------------------------------------ + # Public accessors + # ------------------------------------------------------------------ + + def get_recent_events(self, count=10): + """Return the most recent tide events.""" + events = list(self.events) + return events[-count:] if len(events) > count else events + + def get_last_high(self): + """Return the most recent high tide event, or None.""" + for event in reversed(self.events): + if event[2] == "high": + return event + return None + + def get_last_low(self): + """Return the most recent low tide event, or None.""" + for event in reversed(self.events): + if event[2] == "low": + return event + return None diff --git a/dbus-tides/tide_harmonics.py b/dbus-tides/tide_harmonics.py new file mode 100644 index 0000000..4044ff7 --- /dev/null +++ b/dbus-tides/tide_harmonics.py @@ -0,0 +1,384 @@ +""" +Pure-Python tidal harmonic prediction engine. + +Implements the IHO standard harmonic method for predicting ocean tides. +Zero external dependencies -- uses only the Python standard library. + +Reference: Schureman, "Manual of Harmonic Analysis and Prediction of Tides" + (US Coast and Geodetic Survey, Special Publication No. 98) + +The prediction equation: + h(t) = Z0 + sum_i( f_i * A_i * cos(V_i(t) + u_i - G_i) ) + +Where: + Z0 = mean water level offset + A_i = amplitude of constituent i (from model, location-specific) + G_i = phase (Greenwich epoch) of constituent i (from model) + f_i = nodal amplitude factor (varies over 18.6-year lunar node cycle) + u_i = nodal phase correction (same cycle) + V_i = astronomical argument at time t +""" + +import math + +DEG2RAD = math.pi / 180.0 +RAD2DEG = 180.0 / math.pi + +# ===================================================================== +# TIDAL CONSTITUENT DEFINITIONS +# ===================================================================== +# +# 37 standard tidal constituents. +# Each: (name, Doodson_numbers, speed_deg_per_hour) +# Doodson numbers: (T, s, h, p, N_prime, ps) +# Speeds from Schureman Table 2. + +CONSTITUENTS = [ + # Semi-diurnal + ("M2", (2, -2, 2, 0, 0, 0), 28.9841042), + ("S2", (2, 0, 0, 0, 0, 0), 30.0000000), + ("N2", (2, -3, 2, 1, 0, 0), 28.4397295), + ("K2", (2, 0, 2, 0, 0, 0), 30.0821373), + ("2N2", (2, -4, 2, 2, 0, 0), 27.8953548), + ("MU2", (2, -4, 4, 0, 0, 0), 27.9682084), + ("NU2", (2, -3, 4, -1, 0, 0), 28.5125831), + ("L2", (2, -1, 2, -1, 0, 0), 29.5284789), + ("T2", (2, 0, -1, 0, 0, 1), 29.9589333), + ("R2", (2, 0, 1, 0, 0, -1), 30.0410667), + ("LAM2", (2, -1, 0, 1, 0, 0), 29.4556253), + # Diurnal + ("K1", (1, 0, 1, 0, 0, 0), 15.0410686), + ("O1", (1, -2, 1, 0, 0, 0), 13.9430356), + ("P1", (1, 0, -1, 0, 0, 0), 14.9589314), + ("Q1", (1, -3, 1, 1, 0, 0), 13.3986609), + ("J1", (1, 1, 1, -1, 0, 0), 15.5854433), + ("OO1", (1, 2, 1, 0, 0, 0), 16.1391017), + ("M1", (1, -1, 1, 0, 0, 0), 14.4920521), + ("2Q1", (1, -4, 1, 2, 0, 0), 12.8542862), + ("RHO1", (1, -3, 3, -1, 0, 0), 13.4715145), + # Long-period + ("MF", (0, 2, 0, 0, 0, 0), 1.0980331), + ("MM", (0, 1, 0, -1, 0, 0), 0.5443747), + ("SSA", (0, 0, 2, 0, 0, 0), 0.0821373), + ("MSM", (0, 1, -2, 1, 0, 0), 0.4715211), + ("MS", (0, 2, -2, 0, 0, 0), 1.0158958), + # Shallow-water / compound + ("M3", (3, -3, 3, 0, 0, 0), 43.4761563), + ("M4", (4, -4, 4, 0, 0, 0), 57.9682084), + ("MS4", (4, -2, 2, 0, 0, 0), 58.9841042), + ("M6", (6, -6, 6, 0, 0, 0), 86.9523127), + ("M8", (8, -8, 8, 0, 0, 0), 115.9364168), + ("MN4", (4, -5, 4, 1, 0, 0), 57.4238337), + ("S4", (4, 0, 0, 0, 0, 0), 60.0000000), + ("S6", (6, 0, 0, 0, 0, 0), 90.0000000), + ("2SM2", (2, 2, -2, 0, 0, 0), 31.0158958), + ("MKS2", (2, -2, 4, 0, 0, 0), 29.0662415), + # Additional + ("S1", (1, 0, 0, 0, 0, 0), 15.0000000), + ("SA", (0, 0, 1, 0, 0, 0), 0.0410686), + # Terdiurnal + ("MK3", (3, -2, 3, 0, 0, 0), 44.0251729), + ("2MK3", (3, -4, 3, 0, 0, 0), 42.9271398), +] + +_CONST_BY_NAME = {c[0]: (c[1], c[2]) for c in CONSTITUENTS} +CONSTITUENT_NAMES = [c[0] for c in CONSTITUENTS] + + +# ===================================================================== +# ASTRONOMICAL ARGUMENTS +# ===================================================================== + +def _julian_centuries(unix_ts): + """Convert Unix timestamp to Julian centuries from J2000.0.""" + jd = unix_ts / 86400.0 + 2440587.5 + return (jd - 2451545.0) / 36525.0 + + +def _astro_angles(T): + """Compute fundamental astronomical angles in degrees. + + Args: + T: Julian centuries from J2000.0 + + Returns: + (s, h, p, N, ps) all in degrees + + s = mean longitude of the Moon + h = mean longitude of the Sun + p = mean longitude of lunar perigee + N = longitude of Moon's ascending node + ps = mean longitude of solar perigee + + Formulae from Meeus (1998), consistent with Schureman/IHO. + """ + s = (218.3164477 + + 481267.88123421 * T + - 0.0015786 * T * T + + T * T * T / 538841.0 + - T * T * T * T / 65194000.0) + + h = 280.46646 + 36000.76983 * T + 0.0003032 * T * T + + p = (83.3532465 + + 4069.0137287 * T + - 0.0103200 * T * T + - T * T * T / 80053.0 + + T * T * T * T / 18999000.0) + + N = (125.04452 + - 1934.13626 * T + + 0.0020708 * T * T + + T * T * T / 450000.0) + + ps = 282.93735 + 1.71946 * T + 0.00046 * T * T + + return s, h, p, N, ps + + +def _gmst_degrees(T): + """Greenwich Mean Sidereal Time in degrees. + + Args: + T: Julian centuries from J2000.0 + + Meeus (1998) eq. 12.4, valid for the 21st century. + """ + D = T * 36525.0 + return (280.46061837 + 360.98564736629 * D + + 0.000387933 * T * T + - T * T * T / 38710000.0) % 360.0 + + +def _solar_hour_angle(theta_g, h): + """Mean solar hour angle at Greenwich (degrees). + + T = GMST - h, where h is mean longitude of the Sun. + The first Doodson number multiplies T (not lunar time tau). + Verified: speed(M2) = 2*(GMST_rate - h_rate) - 2*s_rate + 2*h_rate + = 2*GMST_rate - 2*s_rate = 28.984 deg/hr ✓ + """ + return theta_g - h + + +def _V0(doodson, T_solar, s, h, p, N, ps): + """Astronomical argument V0 for a constituent (degrees).""" + dT, ds, dh, dp, dN, dps = doodson + return dT * T_solar + ds * s + dh * h + dp * p + dN * N + dps * ps + + +# ===================================================================== +# NODAL CORRECTIONS (Schureman 1958) +# ===================================================================== + +def _nodal_corrections(N_deg): + """Compute nodal f (amplitude factor) and u (phase correction, degrees). + + Args: + N_deg: longitude of Moon's ascending node in degrees + + Returns: + dict {constituent_name: (f, u_degrees)} + """ + N = N_deg * DEG2RAD + cosN = math.cos(N) + cos2N = math.cos(2.0 * N) + sinN = math.sin(N) + sin2N = math.sin(2.0 * N) + + corrections = {} + + # M2 family (semi-diurnal lunar) + f_m2 = 1.0 - 0.03690 * cosN + 0.00256 * cos2N + u_m2 = -2.14 * sinN + 0.29 * sin2N + for name in ("M2", "N2", "2N2", "MU2", "NU2", "LAM2", "2SM2", "MKS2"): + corrections[name] = (f_m2, u_m2) + + # S2 family (solar, no nodal dependence) + for name in ("S2", "T2", "R2"): + corrections[name] = (1.0, 0.0) + + # K2 + f_k2 = 1.0 + 0.2852 * cosN + 0.0324 * cos2N + u_k2 = -17.74 * sinN + 0.68 * sin2N + corrections["K2"] = (f_k2, u_k2) + + # L2 + lr = 1.0 - 0.25 * cosN + li = 0.25 * sinN + f_l2 = math.sqrt(lr * lr + li * li) + u_l2 = math.atan2(li, lr) * RAD2DEG + corrections["L2"] = (f_l2, u_l2) + + # K1 + kr = 1.0 + 0.1158 * cosN - 0.0029 * cos2N + ki = 0.1554 * sinN - 0.0029 * sin2N + f_k1 = math.sqrt(kr * kr + ki * ki) + u_k1 = -math.atan2(ki, kr) * RAD2DEG + corrections["K1"] = (f_k1, u_k1) + corrections["J1"] = (f_k1, u_k1) + corrections["OO1"] = (f_k1, u_k1) + + # O1 family + o_r = 1.0 + 0.1886 * cosN - 0.0058 * cos2N + o_i = 0.1886 * sinN - 0.0058 * sin2N + f_o1 = math.sqrt(o_r * o_r + o_i * o_i) + u_o1 = math.atan2(o_i, o_r) * RAD2DEG + for name in ("O1", "Q1", "2Q1", "RHO1", "M1"): + corrections[name] = (f_o1, u_o1) + + # P1, S1 -- no nodal + corrections["P1"] = (1.0, 0.0) + corrections["S1"] = (1.0, 0.0) + + # Long-period + f_mf = 1.043 + 0.414 * cosN + u_mf = -23.74 * sinN + 2.68 * sin2N + corrections["MF"] = (f_mf, u_mf) + + f_mm = 1.0 - 0.130 * cosN + corrections["MM"] = (f_mm, 0.0) + corrections["MSM"] = (f_mm, 0.0) + corrections["MS"] = (f_m2, u_m2) + + for name in ("SSA", "SA"): + corrections[name] = (1.0, 0.0) + + # Shallow water + corrections["M3"] = (f_m2 ** 1.5, 1.5 * u_m2) + corrections["M4"] = (f_m2 * f_m2, 2.0 * u_m2) + corrections["MN4"] = (f_m2 * f_m2, 2.0 * u_m2) + corrections["MS4"] = (f_m2, u_m2) + corrections["M6"] = (f_m2 ** 3, 3.0 * u_m2) + corrections["M8"] = (f_m2 ** 4, 4.0 * u_m2) + corrections["S4"] = (1.0, 0.0) + corrections["S6"] = (1.0, 0.0) + + # Terdiurnal + corrections["MK3"] = (f_m2 * f_k1, u_m2 + u_k1) + corrections["2MK3"] = (f_m2 * f_m2 * f_k1, 2.0 * u_m2 + u_k1) + + return corrections + + +# ===================================================================== +# PREDICTION +# ===================================================================== + +def predict(constituents, timestamps, z0=0.0): + """Predict tide heights at the given timestamps. + + Args: + constituents: list of dicts with keys: + "name" - constituent name (e.g. "M2") + "amplitude" - amplitude in meters + "phase" - Greenwich phase in degrees + timestamps: list of Unix timestamps (float, seconds since epoch) + z0: mean water level offset (meters) + + Returns: + list of predicted tide heights (meters) + """ + if not constituents or not timestamps: + return [] + + t0 = timestamps[0] + T0 = _julian_centuries(t0) + s, h, p, N, ps = _astro_angles(T0) + theta_g = _gmst_degrees(T0) + T_solar = _solar_hour_angle(theta_g, h) + nodal = _nodal_corrections(N) + + precomp = [] + for c in constituents: + name = c["name"] + amp = c["amplitude"] + phase_g = c["phase"] + + if name not in _CONST_BY_NAME: + continue + if amp <= 0.0: + continue + + doodson, speed = _CONST_BY_NAME[name] + f, u = nodal.get(name, (1.0, 0.0)) + + v0 = _V0(doodson, T_solar, s, h, p, N, ps) + speed_rad = speed * DEG2RAD + phase_offset = (v0 + u - phase_g) * DEG2RAD + + precomp.append((amp * f, speed_rad, phase_offset)) + + heights = [] + for ts in timestamps: + dt_hr = (ts - t0) / 3600.0 + total = z0 + for amp_f, speed_rad, phase_offset in precomp: + total += amp_f * math.cos(speed_rad * dt_hr + phase_offset) + heights.append(total) + + return heights + + +def find_extrema(timestamps, heights, min_amplitude=0.3): + """Find high and low tide events from a predicted curve. + + Uses a multi-pass algorithm to avoid the cascading-rejection problem + that occurs with single-pass filtering when diurnal inequality creates + small-range tidal cycles (common in the Caribbean, Gulf of Mexico, etc.). + + Pipeline: + 1. Identify all local maxima and minima + 2. Merge consecutive same-type extrema (enforce strict alternation) + 3. Iteratively remove the adjacent pair with the smallest height + range until all remaining pairs exceed min_amplitude + + Args: + timestamps: list of Unix timestamps + heights: corresponding predicted heights + min_amplitude: minimum height change between adjacent extrema + + Returns: + list of (timestamp, height, "high"|"low") tuples + """ + if len(timestamps) < 3: + return [] + + candidates = [] + for i in range(1, len(heights) - 1): + hp = heights[i - 1] + hc = heights[i] + hn = heights[i + 1] + + if hc > hp and hc > hn: + candidates.append((timestamps[i], hc, "high")) + elif hc < hp and hc < hn: + candidates.append((timestamps[i], hc, "low")) + + if not candidates: + return [] + + merged = [candidates[0]] + for ts, h, t in candidates[1:]: + if merged[-1][2] == t: + _, prev_h, _ = merged[-1] + if (t == "high" and h > prev_h) or (t == "low" and h < prev_h): + merged[-1] = (ts, h, t) + else: + merged.append((ts, h, t)) + + changed = True + while changed and len(merged) > 2: + changed = False + min_range = float('inf') + min_idx = -1 + for i in range(len(merged) - 1): + r = abs(merged[i][1] - merged[i + 1][1]) + if r < min_range: + min_range = r + min_idx = i + if min_range < min_amplitude and min_idx >= 0: + merged = merged[:min_idx] + merged[min_idx + 2:] + changed = True + + return merged diff --git a/dbus-tides/tide_merger.py b/dbus-tides/tide_merger.py new file mode 100644 index 0000000..8144af1 --- /dev/null +++ b/dbus-tides/tide_merger.py @@ -0,0 +1,337 @@ +""" +Tide merger -- calibrates model predictions against observed depth. + +The model predicts tide height relative to mean sea level. Observed +depth is measured from the transducer to the seabed. The merger +computes a "chart depth" offset so that: + + predicted_depth = chart_depth + predicted_tide_height + +Where chart_depth is calibrated from: + + chart_depth = mean(observed_depth) - mean(predicted_tide_height) + +This gives us predicted depths under the keel for the current location. +When the vessel moves more than CHART_DEPTH_RECAL_METERS, chart_depth +is recalibrated from recent observations at the new position. +""" + +import logging +import math +import time + +from config import CHART_DEPTH_RECAL_METERS + +logger = logging.getLogger('TideMerger') + + +def _haversine(lat1, lon1, lat2, lon2): + """Great-circle distance in meters between two GPS coordinates.""" + R = 6371000 + phi1 = math.radians(lat1) + phi2 = math.radians(lat2) + dphi = math.radians(lat2 - lat1) + dlam = math.radians(lon2 - lon1) + a = (math.sin(dphi / 2) ** 2 + + math.cos(phi1) * math.cos(phi2) * math.sin(dlam / 2) ** 2) + return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) + + +class TideMerger: + """Merges observed depth with model predictions.""" + + def __init__(self): + self._chart_depth = None + self._calibrated = False + self._calibration_samples = 0 + self._cal_lat = None + self._cal_lon = None + + @property + def is_calibrated(self): + return self._calibrated + + @property + def chart_depth(self): + return self._chart_depth + + def needs_recalibration(self, lat, lon): + """Check if chart_depth needs recalibrating due to a position change.""" + if lat is None or lon is None: + return False + if self._cal_lat is None or self._cal_lon is None: + return False + return _haversine(lat, lon, self._cal_lat, self._cal_lon) > CHART_DEPTH_RECAL_METERS + + def calibrate(self, observed_history, predicted_ts, predicted_heights, + lat=None, lon=None): + """Calibrate chart_depth from overlapping observed and predicted data. + + When lat/lon are provided and the vessel has moved more than + CHART_DEPTH_RECAL_METERS from the last calibration position, + only recent observations (last 30 minutes) are used so that + chart_depth reflects the new seabed depth immediately. + + Args: + observed_history: list of (timestamp, depth) from DepthRecorder + predicted_ts: list of prediction timestamps + predicted_heights: list of predicted tide heights (MSL-relative) + lat, lon: current vessel position + + Returns: + True if calibration succeeded. + """ + if not observed_history or not predicted_ts or not predicted_heights: + return False + + obs_to_use = observed_history + + if lat is not None and lon is not None: + use_recent = False + if self._cal_lat is None or self._cal_lon is None: + use_recent = True + reason = "initial calibration (no prior position)" + else: + dist = _haversine(lat, lon, self._cal_lat, self._cal_lon) + if dist > CHART_DEPTH_RECAL_METERS: + use_recent = True + reason = ("position moved %.0fm from calibration point" + % dist) + + if use_recent: + now = time.time() + recent = [(ts, d) for ts, d in observed_history + if now - ts < 1800] + if len(recent) >= 3: + obs_to_use = recent + logger.info( + "%s, calibrating chart_depth from %d recent obs", + reason, len(recent)) + elif self._cal_lat is not None: + logger.info( + "%s but only %d recent obs, " + "keeping current chart_depth", + reason, len(recent)) + return False + + pred_start = predicted_ts[0] + pred_end = predicted_ts[-1] + + overlapping_obs = [ + (ts, d) for ts, d in obs_to_use + if pred_start <= ts <= pred_end + ] + + if len(overlapping_obs) < 6: + if obs_to_use: + mean_obs = sum(d for _, d in obs_to_use) / len(obs_to_use) + mean_pred = sum(predicted_heights) / len(predicted_heights) + self._chart_depth = mean_obs - mean_pred + self._calibration_samples = len(obs_to_use) + self._calibrated = True + self._cal_lat = lat + self._cal_lon = lon + logger.info( + "Rough calibration: chart_depth=%.2f (from %d obs, %d pred)", + self._chart_depth, len(obs_to_use), + len(predicted_heights)) + return True + return False + + mean_obs = sum(d for _, d in overlapping_obs) / len(overlapping_obs) + + interp_pred = [] + for obs_ts, _ in overlapping_obs: + h = self._interp(obs_ts, predicted_ts, predicted_heights) + if h is not None: + interp_pred.append(h) + + if not interp_pred: + return False + + mean_pred = sum(interp_pred) / len(interp_pred) + self._chart_depth = mean_obs - mean_pred + self._calibration_samples = len(overlapping_obs) + self._calibrated = True + self._cal_lat = lat + self._cal_lon = lon + + logger.info( + "Calibrated: chart_depth=%.2f (from %d overlapping samples)", + self._chart_depth, len(overlapping_obs)) + return True + + def reset_calibration(self): + """Reset calibration when vessel moves to a new location.""" + self._chart_depth = None + self._calibrated = False + self._calibration_samples = 0 + self._cal_lat = None + self._cal_lon = None + logger.info("Calibration reset (location change)") + + def merge_predictions(self, predicted_ts, predicted_heights): + """Convert MSL-relative predictions to depth-under-keel predictions. + + Args: + predicted_ts: list of timestamps + predicted_heights: list of MSL-relative heights + + Returns: + list of predicted depths under keel, or None if not calibrated. + """ + if not self._calibrated or self._chart_depth is None: + return None + return [self._chart_depth + h for h in predicted_heights] + + def get_next_tides(self, extrema, count=2): + """Get the next N high and low tides from predicted extrema. + + Args: + extrema: list of (timestamp, height, "high"|"low") + count: how many of each type to return + + Returns: + dict with keys: + 'high': [(ts, depth), ...] -- next `count` highs + 'low': [(ts, depth), ...] -- next `count` lows + """ + now = time.time() + future = [(ts, h, t) for ts, h, t in extrema if ts > now] + + highs = [] + lows = [] + for ts, h, tide_type in future: + if tide_type == "high" and len(highs) < count: + highs.append((ts, h)) + elif tide_type == "low" and len(lows) < count: + lows.append((ts, h)) + if len(highs) >= count and len(lows) >= count: + break + + return {'high': highs, 'low': lows} + + def get_prev_tides(self, extrema, count=1): + """Get the most recent N past high and low tides from predicted extrema. + + Args: + extrema: list of (timestamp, height, "high"|"low") + count: how many of each type to return + + Returns: + dict with keys: + 'high': [(ts, depth), ...] -- most recent `count` past highs + 'low': [(ts, depth), ...] -- most recent `count` past lows + """ + now = time.time() + past = [(ts, h, t) for ts, h, t in extrema if ts <= now] + + highs = [] + lows = [] + for ts, h, tide_type in reversed(past): + if tide_type == "high" and len(highs) < count: + highs.append((ts, h)) + elif tide_type == "low" and len(lows) < count: + lows.append((ts, h)) + if len(highs) >= count and len(lows) >= count: + break + + return {'high': highs, 'low': lows} + + def build_combined_curve(self, observed_history, predicted_ts, + predicted_depths, hours_back=24): + """Build a combined depth curve: observed trailing + predicted future. + + Args: + observed_history: list of (timestamp, depth) + predicted_ts: list of predicted timestamps + predicted_depths: list of predicted depths (calibrated) + hours_back: how many hours of observed history to include + + Returns: + dict with 'observed' and 'predicted' arrays of {ts, depth}. + """ + now = time.time() + cutoff = now - hours_back * 3600 + + obs_points = [ + {'ts': int(ts), 'depth': round(d, 2)} + for ts, d in observed_history + if ts > cutoff + ] + + pred_points = [] + if predicted_ts and predicted_depths: + for ts, d in zip(predicted_ts, predicted_depths): + pred_points.append({ + 'ts': int(ts), + 'depth': round(d, 2), + }) + + return { + 'observed': obs_points, + 'predicted': pred_points, + } + + def build_dual_curves(self, observed_history, station_ts, station_depths, + local_ts=None, local_depths=None, hours_back=24): + """Build observed + station + local prediction curves. + + Args: + observed_history: list of (timestamp, depth) + station_ts: prediction timestamps (unadjusted model) + station_depths: calibrated station depths + local_ts: adapted prediction timestamps (or None) + local_depths: adapted calibrated depths (or None) + hours_back: hours of observed history to include + + Returns: + dict with 'observed', 'station', and 'local' arrays of + {ts, depth}. + """ + now = time.time() + cutoff = now - hours_back * 3600 + + obs_points = [ + {'ts': int(ts), 'depth': round(d, 2)} + for ts, d in observed_history + if ts > cutoff + ] + + station_points = [] + if station_ts and station_depths: + for ts, d in zip(station_ts, station_depths): + station_points.append( + {'ts': int(ts), 'depth': round(d, 2)}) + + local_points = [] + if local_ts and local_depths: + for ts, d in zip(local_ts, local_depths): + local_points.append( + {'ts': int(ts), 'depth': round(d, 2)}) + + return { + 'observed': obs_points, + 'station': station_points, + 'local': local_points, + } + + @staticmethod + def _interp(target_ts, timestamps, values): + """Linear interpolation of a value at target_ts.""" + if not timestamps: + return None + if target_ts <= timestamps[0]: + return values[0] + if target_ts >= timestamps[-1]: + return values[-1] + + for i in range(len(timestamps) - 1): + if timestamps[i] <= target_ts <= timestamps[i + 1]: + dt = timestamps[i + 1] - timestamps[i] + if dt == 0: + return values[i] + frac = (target_ts - timestamps[i]) / dt + return values[i] + frac * (values[i + 1] - values[i]) + + return None diff --git a/dbus-tides/tide_predictor.py b/dbus-tides/tide_predictor.py new file mode 100644 index 0000000..7db50a8 --- /dev/null +++ b/dbus-tides/tide_predictor.py @@ -0,0 +1,824 @@ +""" +Tide prediction orchestrator. + +Loads tidal constituent data from the bundled coastal grid (or fallback +sources), interpolates for the vessel position, and generates tide +predictions using the pure-Python harmonic engine. +""" + +import gzip +import json +import logging +import math +import os +import time + +from config import ( + BESTFIT_MIN_HISTORY_HOURS, + CONSTITUENTS_DIR, + DATA_DIR, + GRID_FILE_NAME, + NOAA_NEARBY_COUNT, + NOAA_STATION_MAX_DISTANCE, + NOAA_STATIONS_FILE_NAME, + PREDICTION_HOURS, + PREDICTION_INTERVAL, +) +import tide_harmonics + +# Harmonic predictions are smooth mathematical curves with no noise, +# so extrema filtering only needs to suppress shallow-water harmonic +# artifacts (typically < 1 cm). The observed-data threshold +# (MIN_TIDE_AMPLITUDE) is far too aggressive here and drops +# legitimate small-range tidal cycles caused by diurnal inequality. +_PREDICTION_EXTREMA_MIN = 0.02 + +logger = logging.getLogger('TidePredictor') + + +def _find_subordinate_ref_extrema(timestamps, heights, min_amplitude=0.4): + """Single-pass extrema finder for subordinate reference curves. + + The subordinate height-correction factor (h_high / h_low) is + interpolated between reference-curve extrema. The multi-pass + pair-removal in find_extrema() selects different surviving extrema, + shifting factor boundaries and distorting the subordinate curve. + This preserves the original sequential logic so the curve shape + remains stable. + """ + if len(timestamps) < 3: + return [] + extrema = [] + last_h = None + for i in range(1, len(heights) - 1): + hp = heights[i - 1] + hc = heights[i] + hn = heights[i + 1] + if hc > hp and hc > hn: + etype = "high" + elif hc < hp and hc < hn: + etype = "low" + else: + continue + if last_h is not None and abs(hc - last_h) < min_amplitude: + continue + extrema.append((timestamps[i], hc, etype)) + last_h = hc + return extrema + + +class TidePredictor: + """Manages constituent data and produces tide predictions.""" + + def __init__(self): + self._grid = None + self._grid_loaded = False + self._noaa_stations = None + self._noaa_loaded = False + self._current_constituents = None + self._current_source = None + self._current_station = None + self._current_offsets = None + self._nearby_stations = [] + self._station_override = None + self._current_lat = None + self._current_lon = None + self._last_prediction = None + self._last_extrema = None + self._last_prediction_time = 0 + + # ------------------------------------------------------------------ + # NOAA station loading + # ------------------------------------------------------------------ + + def load_noaa_stations(self): + """Load the pre-built NOAA station database from disk.""" + if self._noaa_loaded: + return self._noaa_stations is not None + + self._noaa_loaded = True + path = self._find_noaa_file() + if not path: + logger.info("No NOAA station file found") + return False + + try: + if path.endswith('.gz'): + with gzip.open(path, 'rt', encoding='utf-8') as f: + self._noaa_stations = json.load(f) + else: + with open(path, 'r') as f: + self._noaa_stations = json.load(f) + + n = len(self._noaa_stations.get('stations', [])) + logger.info("Loaded NOAA station database: %d stations", n) + return True + except (OSError, json.JSONDecodeError) as e: + logger.error("Failed to load NOAA stations: %s", e) + self._noaa_stations = None + return False + + def _find_noaa_file(self): + """Search for the NOAA station file in known locations.""" + candidates = [ + os.path.join(DATA_DIR, 'constituents', NOAA_STATIONS_FILE_NAME), + os.path.join(CONSTITUENTS_DIR, NOAA_STATIONS_FILE_NAME), + os.path.join(os.path.dirname(__file__), 'constituents', + NOAA_STATIONS_FILE_NAME), + os.path.join(os.path.dirname(__file__), 'constituents', + NOAA_STATIONS_FILE_NAME.replace('.gz', '')), + ] + for path in candidates: + if os.path.exists(path): + logger.info("Found NOAA station file: %s", path) + return path + return None + + def _find_nearby_noaa_stations(self, lat, lon): + """Find the N nearest NOAA stations within range. + + Populates self._nearby_stations with (distance, station) pairs + for all stations within NOAA_STATION_MAX_DISTANCE, sorted by + distance (nearest first), up to NOAA_NEARBY_COUNT. + """ + self._nearby_stations = [] + if not self.load_noaa_stations() or self._noaa_stations is None: + return + + stations = self._noaa_stations.get('stations', []) + if not stations: + return + + dists = [] + for st in stations: + d = _haversine(lat, lon, st['lat'], st['lon']) + if d <= NOAA_STATION_MAX_DISTANCE: + dists.append((d, st)) + dists.sort(key=lambda x: x[0]) + self._nearby_stations = dists[:NOAA_NEARBY_COUNT] + + if self._nearby_stations: + logger.info( + "Found %d NOAA stations within %.0f nm", + len(self._nearby_stations), + NOAA_STATION_MAX_DISTANCE / 1852) + for d, st in self._nearby_stations: + stype = ' (sub)' if st.get('type') == 'S' else '' + logger.info( + " %s %s (%.1f nm)%s", + st.get('id', '?'), st.get('name', '?'), + d / 1852, stype) + else: + logger.info( + "No NOAA stations within %.0f nm of %.2f, %.2f", + NOAA_STATION_MAX_DISTANCE / 1852, lat, lon) + + def _select_noaa_station(self, lat, lon, depth_history=None): + """Select the best NOAA station for the current position. + + Selection priority: + 1. Manual override (if station_override is set and valid) + 2. Best-fit scoring against observed depth history + 3. Nearest station by distance + + Returns a constituent list, or None if no station in range. + """ + self._find_nearby_noaa_stations(lat, lon) + if not self._nearby_stations: + return None + + chosen_dist, chosen_station = None, None + + if self._station_override: + for d, st in self._nearby_stations: + if st.get('id') == self._station_override: + chosen_dist, chosen_station = d, st + logger.info( + "Using overridden station %s (%s)", + st.get('id', '?'), st.get('name', '?')) + break + if chosen_station is None: + all_stations = self._noaa_stations.get('stations', []) + for st in all_stations: + if st.get('id') == self._station_override: + chosen_dist = _haversine( + lat, lon, st['lat'], st['lon']) + chosen_station = st + logger.info( + "Using overridden station %s (%s), %.1f km away", + st.get('id', '?'), st.get('name', '?'), + chosen_dist / 1000) + break + + if chosen_station is None and depth_history: + scored = self._rank_stations_by_fit( + self._nearby_stations, depth_history) + if scored: + chosen_dist, chosen_station = scored[0] + + if chosen_station is None: + chosen_dist, chosen_station = self._nearby_stations[0] + + is_subordinate = chosen_station.get('type') == 'S' + if is_subordinate: + constituents, offsets = self._resolve_subordinate(chosen_station) + else: + constituents = chosen_station.get('constituents', []) + offsets = None + + if not constituents: + return None + + self._current_offsets = offsets + self._current_station = { + 'id': chosen_station.get('id', ''), + 'name': chosen_station.get('name', ''), + 'lat': chosen_station.get('lat', 0.0), + 'lon': chosen_station.get('lon', 0.0), + 'distance': chosen_dist, + } + if is_subordinate: + self._current_station['type'] = 'S' + self._current_station['ref_id'] = chosen_station.get('ref_id', '') + self._current_source = 'noaa:%s' % chosen_station.get('id', '?') + + logger.info( + "Selected NOAA station %s (%s), %.1f km, %d constituents%s", + chosen_station.get('id', '?'), + chosen_station.get('name', '?'), + chosen_dist / 1000, + len(constituents), + ' (subordinate)' if is_subordinate else '') + return constituents + + def _resolve_subordinate(self, sub_station): + """Resolve a subordinate station to its reference station's + constituents and the subordinate offsets. + + Returns (constituents, offsets) or (None, None). + """ + ref_id = sub_station.get('ref_id', '') + if not ref_id or not self._noaa_stations: + return None, None + + ref_station = None + for st in self._noaa_stations.get('stations', []): + if st.get('id') == ref_id and st.get('constituents'): + ref_station = st + break + + if not ref_station: + logger.warning( + "Reference station %s not found for subordinate %s", + ref_id, sub_station.get('id', '?')) + return None, None + + offsets = sub_station.get('offsets', {}) + logger.info( + "Resolved subordinate %s -> reference %s (%s), " + "time_high=%+dmin, time_low=%+dmin, " + "height_high=%.2f, height_low=%.2f (%s)", + sub_station.get('id', '?'), + ref_id, ref_station.get('name', '?'), + offsets.get('time_high', 0), offsets.get('time_low', 0), + offsets.get('height_high', 1.0), offsets.get('height_low', 1.0), + offsets.get('height_type', 'R')) + return ref_station['constituents'], offsets + + def _rank_stations_by_fit(self, candidates, depth_history): + """Score candidate stations against observed depth data. + + Runs a hindcast for each candidate over the observed time window, + then picks the one whose predicted tidal shape best correlates + with the detrended depth observations. + + Returns candidates re-sorted by fit quality (best first), + or None if insufficient history. + """ + if not depth_history or len(depth_history) < 6: + return None + + span_hours = (depth_history[-1][0] - depth_history[0][0]) / 3600.0 + if span_hours < BESTFIT_MIN_HISTORY_HOURS: + return None + + obs_ts = [ts for ts, _ in depth_history] + obs_depths = [d for _, d in depth_history] + + obs_mean = sum(obs_depths) / len(obs_depths) + obs_centered = [d - obs_mean for d in obs_depths] + + scored = [] + for dist, st in candidates: + if st.get('type') == 'S': + constituents, offsets = self._resolve_subordinate(st) + else: + constituents = st.get('constituents', []) + offsets = None + if not constituents: + continue + + if offsets: + time_shift = (offsets.get('time_high', 0) + + offsets.get('time_low', 0)) / 2.0 * 60.0 + shifted_ts = [t - time_shift for t in obs_ts] + raw_heights = tide_harmonics.predict( + constituents, shifted_ts) + if not raw_heights or len(raw_heights) != len(obs_ts): + continue + h_high = offsets.get('height_high', 1.0) + h_low = offsets.get('height_low', 1.0) + h_type = offsets.get('height_type', 'R') + avg_factor = (h_high + h_low) / 2.0 + if h_type == 'R': + pred_heights = [h * avg_factor for h in raw_heights] + else: + pred_heights = [h + avg_factor for h in raw_heights] + else: + pred_heights = tide_harmonics.predict(constituents, obs_ts) + if not pred_heights or len(pred_heights) != len(obs_ts): + continue + + pred_mean = sum(pred_heights) / len(pred_heights) + pred_centered = [h - pred_mean for h in pred_heights] + + corr = _correlation(obs_centered, pred_centered) + scored.append((dist, st, corr)) + + if not scored: + return None + + scored.sort(key=lambda x: -x[2]) + + for dist, st, corr in scored: + stype = ' (sub)' if st.get('type') == 'S' else '' + logger.info( + " Fit score: %s %s = %.3f (%.1f nm)%s", + st.get('id', '?'), st.get('name', '?'), + corr, dist / 1852, stype) + + best_corr = scored[0][2] + if best_corr < 0.3: + logger.info("Best fit correlation %.3f too low, using nearest", + best_corr) + return None + + return [(d, st) for d, st, _ in scored] + + @property + def station_override(self): + return self._station_override + + @station_override.setter + def station_override(self, station_id): + """Set a manual station override. Empty string or None to clear.""" + if station_id: + self._station_override = str(station_id).strip() + logger.info("Station override set: %s", self._station_override) + else: + self._station_override = None + logger.info("Station override cleared") + + # ------------------------------------------------------------------ + # Grid loading + # ------------------------------------------------------------------ + + def load_grid(self): + """Load the pre-computed coastal grid from disk.""" + if self._grid_loaded: + return self._grid is not None + + self._grid_loaded = True + grid_path = self._find_grid_file() + if not grid_path: + logger.warning("No coastal grid file found") + return False + + try: + if grid_path.endswith('.gz'): + with gzip.open(grid_path, 'rt', encoding='utf-8') as f: + self._grid = json.load(f) + else: + with open(grid_path, 'r') as f: + self._grid = json.load(f) + + n_points = len(self._grid.get('points', [])) + regions = self._grid.get('regions', []) + logger.info( + "Loaded coastal grid: %d points, regions: %s", + n_points, regions) + return True + except (OSError, json.JSONDecodeError) as e: + logger.error("Failed to load grid: %s", e) + self._grid = None + return False + + def _find_grid_file(self): + """Search for the grid file in known locations.""" + candidates = [ + os.path.join(DATA_DIR, 'constituents', GRID_FILE_NAME), + os.path.join(CONSTITUENTS_DIR, GRID_FILE_NAME), + os.path.join(os.path.dirname(__file__), 'constituents', GRID_FILE_NAME), + os.path.join(os.path.dirname(__file__), 'constituents', + GRID_FILE_NAME.replace('.gz', '')), + ] + for path in candidates: + if os.path.exists(path): + logger.info("Found grid file: %s", path) + return path + return None + + # ------------------------------------------------------------------ + # Location update + # ------------------------------------------------------------------ + + def update_location(self, lat, lon, depth_history=None): + """Update constituents for a new vessel position. + + Tries data sources in priority order: + 1. NOAA station (observation-derived, best near coast) + 2. Coastal grid (global model, interpolated) + 3. Custom JSON (manual fallback) + + When depth_history is provided and long enough, candidate NOAA + stations are scored against the observations to pick the best fit. + + Returns True if new constituents were loaded. + """ + self._current_lat = lat + self._current_lon = lon + self._current_source = None + self._current_station = None + self._current_offsets = None + + constituents = self._select_noaa_station( + lat, lon, depth_history=depth_history) + if constituents is None: + constituents = self._interpolate_from_grid(lat, lon) + if constituents is not None: + self._current_source = 'grid' + if constituents is None: + constituents = self._load_custom_json() + if constituents is not None: + self._current_source = 'custom' + if constituents is None: + logger.warning("No constituent data for %.2f, %.2f", lat, lon) + return False + + self._current_constituents = constituents + logger.info( + "Loaded %d constituents for %.2f, %.2f (source: %s)", + len(constituents), lat, lon, self._current_source) + return True + + @property + def current_source(self): + """Return the name of the data source for current constituents.""" + return self._current_source + + @property + def current_station(self): + """Return metadata dict for the selected NOAA station, or None.""" + return self._current_station + + @property + def nearby_stations(self): + """Return list of nearby station dicts with distance info.""" + result = [] + for dist, st in self._nearby_stations: + entry = { + 'id': st.get('id', ''), + 'name': st.get('name', ''), + 'lat': st.get('lat', 0.0), + 'lon': st.get('lon', 0.0), + 'distance': round(dist / 1000, 1), + } + if st.get('type') == 'S': + entry['type'] = 'S' + entry['ref_id'] = st.get('ref_id', '') + result.append(entry) + return result + + # ------------------------------------------------------------------ + # Prediction + # ------------------------------------------------------------------ + + def predict(self, from_time=None, duration_hours=None): + """Generate a tide prediction curve. + + Args: + from_time: Unix timestamp for the start of the curve + (default: now). + duration_hours: length of the curve in hours + (default: PREDICTION_HOURS). + + Returns: + (timestamps, heights, extrema) or (None, None, None). + """ + if not self._current_constituents: + return None, None, None + + if from_time is None: + from_time = time.time() + if duration_hours is None: + duration_hours = PREDICTION_HOURS + + n_steps = duration_hours * 3600 // PREDICTION_INTERVAL + timestamps = [ + from_time + i * PREDICTION_INTERVAL + for i in range(n_steps + 1) + ] + + if self._current_offsets: + heights = self._predict_subordinate(timestamps) + else: + heights = tide_harmonics.predict( + self._current_constituents, timestamps) + + if not heights: + return None, None, None + + extrema = tide_harmonics.find_extrema( + timestamps, heights, min_amplitude=_PREDICTION_EXTREMA_MIN) + + self._last_prediction = (timestamps, heights) + self._last_extrema = extrema + self._last_prediction_time = time.time() + + logger.info( + "Prediction: %d points, %d extrema over %dh", + len(timestamps), len(extrema), duration_hours) + + return timestamps, heights, extrema + + def _predict_subordinate(self, timestamps): + """Predict tide heights for a subordinate station. + + Strategy: + 1. Time-shift the entire reference curve by the average of + the high/low time offsets (handles bulk timing difference). + 2. Find extrema in the shifted reference curve. + 3. Linearly interpolate the height correction factor between + successive extrema (h_high at highs, h_low at lows). + 4. Apply the interpolated factor to produce the final curve. + + This yields a smooth continuous curve that matches NOAA's + subordinate methodology at the extrema and transitions + smoothly between them. + """ + offsets = self._current_offsets + time_high_s = offsets.get('time_high', 0) * 60.0 + time_low_s = offsets.get('time_low', 0) * 60.0 + h_high = offsets.get('height_high', 1.0) + h_low = offsets.get('height_low', 1.0) + h_type = offsets.get('height_type', 'R') + + avg_time_shift = (time_high_s + time_low_s) / 2.0 + + shifted_ts = [t - avg_time_shift for t in timestamps] + ref_heights = tide_harmonics.predict( + self._current_constituents, shifted_ts) + if not ref_heights: + return [] + + ref_extrema = _find_subordinate_ref_extrema( + shifted_ts, ref_heights) + + if len(ref_extrema) < 2: + avg_factor = (h_high + h_low) / 2.0 + if h_type == 'R': + return [h * avg_factor for h in ref_heights] + return [h + avg_factor for h in ref_heights] + + ext_times = [e[0] + avg_time_shift for e in ref_extrema] + ext_factors = [h_high if e[2] == 'high' else h_low + for e in ref_extrema] + + result = [] + for i, t in enumerate(timestamps): + if t <= ext_times[0]: + factor = ext_factors[0] + elif t >= ext_times[-1]: + factor = ext_factors[-1] + else: + j = 0 + for k in range(len(ext_times) - 1): + if ext_times[k] <= t < ext_times[k + 1]: + j = k + break + span = ext_times[j + 1] - ext_times[j] + frac = (t - ext_times[j]) / max(span, 1.0) + factor = (ext_factors[j] + + frac * (ext_factors[j + 1] - ext_factors[j])) + + if h_type == 'R': + result.append(ref_heights[i] * factor) + else: + result.append(ref_heights[i] + factor) + + return result + + def predict_at(self, timestamps): + """Evaluate the harmonic model at arbitrary timestamps. + + Returns a list of MSL-relative heights, or None if no constituents. + """ + if not self._current_constituents or not timestamps: + return None + return tide_harmonics.predict(self._current_constituents, timestamps) + + def get_last_prediction(self): + """Return the most recent prediction result.""" + if self._last_prediction: + return (*self._last_prediction, self._last_extrema) + return None, None, None + + def has_constituents(self): + return self._current_constituents is not None + + @property + def last_prediction_time(self): + return self._last_prediction_time + + def estimate_datum_offset(self): + """Estimate MSL-above-MLLW datum offset from constituent amplitudes. + + Runs a 30-day prediction, finds all lows, groups into daily + pairs, averages the lower of each pair (the MLLW definition). + Returns the offset in meters (MSL - MLLW, always >= 0). + """ + if not self._current_constituents: + return 0.0 + + now = time.time() + step = 600 + n_steps = 30 * 86400 // step + timestamps = [now + i * step for i in range(n_steps + 1)] + + heights = tide_harmonics.predict( + self._current_constituents, timestamps) + if not heights: + return 0.0 + + extrema = tide_harmonics.find_extrema( + timestamps, heights, min_amplitude=_PREDICTION_EXTREMA_MIN) + + lows = [h for _, h, t in extrema if t == "low"] + if len(lows) < 2: + return abs(min(lows)) if lows else 0.0 + + lower_lows = [] + for i in range(0, len(lows) - 1, 2): + lower_lows.append(min(lows[i], lows[i + 1])) + if len(lows) % 2 == 1: + lower_lows.append(lows[-1]) + + mllw = sum(lower_lows) / len(lower_lows) + offset = -mllw + logger.info( + "Estimated datum offset (MSL above MLLW): %.3fm", offset) + return max(offset, 0.0) + + # ------------------------------------------------------------------ + # Constituent interpolation + # ------------------------------------------------------------------ + + def _interpolate_from_grid(self, lat, lon): + """Bilinear interpolation of constituents from the coastal grid.""" + if not self.load_grid() or self._grid is None: + return None + + points = self._grid.get('points', []) + const_names = self._grid.get('constituents', []) + if not points or not const_names: + return None + + nearest = self._find_nearest_points(points, lat, lon, count=4) + if not nearest: + return None + + closest_dist = nearest[0][0] + closest_pt = nearest[0][1] + + # Too far from any grid point + if closest_dist > 100000: + logger.warning( + "Nearest grid point is %.0fkm away", closest_dist / 1000) + return None + + # Very close to a single point -- no interpolation needed + if closest_dist < 1000 or len(nearest) < 4: + return self._point_to_constituents(closest_pt, const_names) + + return self._bilinear_interpolate(nearest, const_names) + + def _find_nearest_points(self, points, lat, lon, count=4): + """Find the N nearest ocean grid points by haversine distance.""" + dists = [] + for pt in points: + d = _haversine(lat, lon, pt['lat'], pt['lon']) + dists.append((d, pt)) + dists.sort(key=lambda x: x[0]) + return dists[:count] + + def _bilinear_interpolate(self, nearest, const_names): + """Inverse-distance-weighted interpolation of constituent data. + + Phase interpolation uses circular averaging (cos/sin components) + to avoid wraparound artifacts at 0/360 degrees. + """ + total_weight = 0.0 + n = len(const_names) + weighted_amp = [0.0] * n + weighted_phase_cos = [0.0] * n + weighted_phase_sin = [0.0] * n + + for dist, pt in nearest: + w = 1.0 / max(dist, 1.0) + total_weight += w + + amps = pt.get('amp', []) + phases = pt.get('phase', []) + + for j in range(min(n, len(amps), len(phases))): + weighted_amp[j] += w * amps[j] + ph_rad = phases[j] * tide_harmonics.DEG2RAD + weighted_phase_cos[j] += w * math.cos(ph_rad) + weighted_phase_sin[j] += w * math.sin(ph_rad) + + if total_weight == 0: + return None + + constituents = [] + for j, name in enumerate(const_names): + amp = weighted_amp[j] / total_weight + phase = math.atan2( + weighted_phase_sin[j], weighted_phase_cos[j] + ) * tide_harmonics.RAD2DEG + if phase < 0: + phase += 360.0 + constituents.append({ + 'name': name, + 'amplitude': amp, + 'phase': phase, + }) + + return constituents + + @staticmethod + def _point_to_constituents(pt, const_names): + """Convert a single grid point to a constituent list.""" + amps = pt.get('amp', []) + phases = pt.get('phase', []) + constituents = [] + for j in range(min(len(const_names), len(amps), len(phases))): + constituents.append({ + 'name': const_names[j], + 'amplitude': amps[j], + 'phase': phases[j], + }) + return constituents + + def _load_custom_json(self): + """Try loading a custom constituent JSON file.""" + for base in (DATA_DIR, os.path.dirname(__file__)): + path = os.path.join(base, 'constituents', 'custom.json') + if os.path.exists(path): + try: + with open(path, 'r') as f: + data = json.load(f) + constituents = data.get('constituents', []) + if constituents: + logger.info( + "Loaded %d constituents from %s", + len(constituents), path) + return constituents + except (OSError, json.JSONDecodeError) as e: + logger.warning("Failed to load %s: %s", path, e) + return None + + +def _haversine(lat1, lon1, lat2, lon2): + """Great-circle distance in meters between two GPS coordinates.""" + R = 6371000 + phi1 = math.radians(lat1) + phi2 = math.radians(lat2) + dphi = math.radians(lat2 - lat1) + dlam = math.radians(lon2 - lon1) + a = (math.sin(dphi / 2) ** 2 + + math.cos(phi1) * math.cos(phi2) * math.sin(dlam / 2) ** 2) + return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) + + +def _correlation(xs, ys): + """Pearson correlation coefficient between two centered sequences.""" + n = len(xs) + if n == 0: + return 0.0 + sum_xy = sum(x * y for x, y in zip(xs, ys)) + sum_x2 = sum(x * x for x in xs) + sum_y2 = sum(y * y for y in ys) + denom = math.sqrt(sum_x2 * sum_y2) + if denom < 1e-12: + return 0.0 + return sum_xy / denom diff --git a/dbus-tides/tides.py b/dbus-tides/tides.py new file mode 100644 index 0000000..e60bbf7 --- /dev/null +++ b/dbus-tides/tides.py @@ -0,0 +1,834 @@ +#!/usr/bin/env python3 +""" +dbus-tides -- Local Tide Prediction for Venus OS + +Monitors vessel depth, detects tidal patterns from observations, +predicts future tides using a pure-Python harmonic engine, and +publishes merged data on D-Bus for the venus-html5-app dashboard. +""" + +import json +import logging +import math +import os +import signal +import sys +import time +import threading +from collections import deque + +sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'ext', 'velib_python')) +sys.path.insert(1, '/opt/victronenergy/velib_python') + +try: + from gi.repository import GLib +except ImportError: + print("ERROR: GLib not available. This script must run on Venus OS.") + sys.exit(1) + +try: + import dbus + from dbus.mainloop.glib import DBusGMainLoop + from vedbus import VeDbusService + from settingsdevice import SettingsDevice +except ImportError as e: + print("ERROR: Required module not available: %s" % e) + print("This script must run on Venus OS.") + sys.exit(1) + +from config import ( + SERVICE_NAME, VERSION, + DEPTH_SAMPLE_INTERVAL, DEPTH_HISTORY_HOURS, SPEED_THRESHOLD_MS, + PREDICTION_HOURS, + GPS_SAMPLE_INTERVAL, GPS_HISTORY_SIZE, GPS_HISTORY_SAMPLE_INTERVAL, + STATIONARY_RADIUS_METERS, MODEL_RERUN_RADIUS_METERS, + MODEL_RERUN_INTERVAL, + LOGGING_CONFIG, +) +from depth_recorder import DepthRecorder +from tide_detector import TideDetector +from tide_predictor import TidePredictor +from tide_merger import TideMerger +from tide_adapter import TideAdapter + +BUS_ITEM = "com.victronenergy.BusItem" +SYSTEM_SERVICE = "com.victronenergy.system" + + +def _unwrap(v): + """Minimal dbus value unwrap.""" + if v is None: + return None + if isinstance(v, (dbus.Int16, dbus.Int32, dbus.Int64, + dbus.UInt16, dbus.UInt32, dbus.UInt64, dbus.Byte)): + return int(v) + if isinstance(v, dbus.Double): + return float(v) + if isinstance(v, (dbus.String, dbus.Signature)): + return str(v) + if isinstance(v, dbus.Boolean): + return bool(v) + if isinstance(v, dbus.Array): + return [_unwrap(x) for x in v] if len(v) > 0 else None + if isinstance(v, (dbus.Dictionary, dict)): + return {k: _unwrap(x) for k, x in v.items()} + return v + + +def haversine_distance(lat1, lon1, lat2, lon2): + """Great-circle distance in meters.""" + R = 6371000 + phi1, phi2 = math.radians(lat1), math.radians(lat2) + dphi = math.radians(lat2 - lat1) + dlam = math.radians(lon2 - lon1) + a = (math.sin(dphi / 2) ** 2 + + math.cos(phi1) * math.cos(phi2) * math.sin(dlam / 2) ** 2) + return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) + + +class GpsReader: + """Read GPS position and speed from Venus OS D-Bus.""" + + def __init__(self, bus): + self.bus = bus + self._gps_service = None + self._proxy_lat = None + self._proxy_lon = None + self._proxy_speed = None + + def _get_proxy(self, service, path): + try: + obj = self.bus.get_object(service, path, introspect=False) + return dbus.Interface(obj, BUS_ITEM) + except dbus.exceptions.DBusException: + return None + + def _refresh_gps_service(self): + if self._gps_service: + return True + try: + proxy = self._get_proxy(SYSTEM_SERVICE, "/GpsService") + if proxy: + svc = _unwrap(proxy.GetValue()) + if svc and isinstance(svc, str): + self._gps_service = svc + self._proxy_lat = self._get_proxy(svc, "/Position/Latitude") + self._proxy_lon = self._get_proxy(svc, "/Position/Longitude") + self._proxy_speed = self._get_proxy(svc, "/Speed") + if not self._proxy_lat: + self._proxy_lat = self._get_proxy(svc, "/Latitude") + if not self._proxy_lon: + self._proxy_lon = self._get_proxy(svc, "/Longitude") + return True + except dbus.exceptions.DBusException: + pass + return False + + def get_position(self): + """Return (lat, lon) or None.""" + if not self._refresh_gps_service(): + return None + lat, lon = None, None + try: + if self._proxy_lat: + lat = _unwrap(self._proxy_lat.GetValue()) + if self._proxy_lon: + lon = _unwrap(self._proxy_lon.GetValue()) + except dbus.exceptions.DBusException: + self._gps_service = None + return None + if (lat is not None and lon is not None and + -90 <= float(lat) <= 90 and -180 <= float(lon) <= 180): + return (float(lat), float(lon)) + return None + + def get_speed(self): + """Return speed in m/s, or None.""" + if not self._refresh_gps_service(): + return None + try: + if self._proxy_speed: + v = _unwrap(self._proxy_speed.GetValue()) + if v is not None: + return float(v) + except dbus.exceptions.DBusException: + self._gps_service = None + return None + + +class DepthDbusReader: + """Read depth from Venus OS D-Bus navigation service.""" + + def __init__(self, bus): + self.bus = bus + self._nav_service = None + self._proxy_depth = None + + def _get_proxy(self, service, path): + try: + obj = self.bus.get_object(service, path, introspect=False) + return dbus.Interface(obj, BUS_ITEM) + except dbus.exceptions.DBusException: + return None + + def _refresh_nav_service(self): + if self._nav_service: + return True + try: + bus_obj = self.bus.get_object( + 'org.freedesktop.DBus', '/org/freedesktop/DBus') + iface = dbus.Interface(bus_obj, 'org.freedesktop.DBus') + names = iface.ListNames() + for name in names: + name_str = str(name) + if name_str.startswith('com.victronenergy.navigation.'): + self._nav_service = name_str + self._proxy_depth = self._get_proxy(name_str, "/Depth") + return True + except dbus.exceptions.DBusException: + pass + return False + + def get_depth(self): + """Return depth in meters, or None.""" + if not self._refresh_nav_service(): + return None + try: + if self._proxy_depth: + v = _unwrap(self._proxy_depth.GetValue()) + if v is not None and float(v) > 0: + return float(v) + except dbus.exceptions.DBusException: + self._nav_service = None + return None + + +class MovementDetector: + """Determines whether the vessel has moved beyond a threshold.""" + + def __init__(self, history_size, threshold_meters): + self.history_size = history_size + self.threshold_meters = threshold_meters + self.positions = deque(maxlen=history_size) + self.last_sample_time = 0 + + def add_position(self, lat, lon, now=None): + if now is None: + now = time.time() + self.positions.append((now, lat, lon)) + self.last_sample_time = now + + def is_moving(self, current_lat, current_lon): + if not self.positions: + return False + for _, hist_lat, hist_lon in self.positions: + dist = haversine_distance(hist_lat, hist_lon, + current_lat, current_lon) + if dist >= self.threshold_meters: + return True + return False + + +class TidesController: + """Coordinates all tide components and D-Bus publishing.""" + + def __init__(self): + self._setup_logging() + self.logger = logging.getLogger('Tides') + self.logger.info("Initializing dbus-tides v%s", VERSION) + + self.bus = dbus.SystemBus() + self._create_dbus_service() + self._setup_settings() + + self.gps = GpsReader(self.bus) + self.depth_reader = DepthDbusReader(self.bus) + self.depth_recorder = DepthRecorder() + self.tide_detector = TideDetector() + self.tide_predictor = TidePredictor() + self.tide_merger = TideMerger() + self.tide_adapter = TideAdapter() + + self.movement_stationary = MovementDetector( + GPS_HISTORY_SIZE, STATIONARY_RADIUS_METERS) + self.movement_model = MovementDetector( + GPS_HISTORY_SIZE, MODEL_RERUN_RADIUS_METERS) + + self.current_lat = None + self.current_lon = None + self.model_lat = None + self.model_lon = None + self.last_depth_sample = 0 + self.last_gps_check = 0 + self.last_gps_history_sample = 0 + self.last_prediction_run = 0 + self.last_event_refresh = 0 + self._predict_in_progress = False + + GLib.timeout_add(1000, self._main_loop) + self.logger.info("Initialized -- sampling depth every %ds", + DEPTH_SAMPLE_INTERVAL) + + def _setup_logging(self): + level = getattr(logging, LOGGING_CONFIG['level'], logging.INFO) + fmt = ('%(asctime)s %(levelname)s %(name)s: %(message)s' + if LOGGING_CONFIG['include_timestamp'] + else '%(levelname)s %(name)s: %(message)s') + logging.basicConfig(level=level, format=fmt, stream=sys.stdout) + + def _create_dbus_service(self): + self.logger = logging.getLogger('Tides') + self.logger.info("Creating D-Bus service: %s", SERVICE_NAME) + + max_retries = 5 + retry_delay = 1.0 + for attempt in range(max_retries): + try: + self.dbus_service = VeDbusService( + SERVICE_NAME, self.bus, register=False) + break + except dbus.exceptions.NameExistsException: + if attempt < max_retries - 1: + self.logger.warning( + "D-Bus name exists, retrying in %.1fs (%d/%d)", + retry_delay, attempt + 1, max_retries) + time.sleep(retry_delay) + retry_delay *= 2 + else: + raise + + svc = self.dbus_service + svc.add_path('/Mgmt/ProcessName', 'dbus-tides') + svc.add_path('/Mgmt/ProcessVersion', VERSION) + svc.add_path('/Mgmt/Connection', 'local') + svc.add_path('/DeviceInstance', 0) + svc.add_path('/ProductId', 0xA162) + svc.add_path('/ProductName', 'Tide Prediction') + svc.add_path('/FirmwareVersion', VERSION) + svc.add_path('/Connected', 1) + + def _status_text(p, v): + labels = {0: 'Idle', 1: 'Calibrating', 2: 'Ready', 3: 'Error'} + return labels.get(v, 'Unknown') if v is not None else 'Unknown' + + svc.add_path('/Status', 0, gettextcallback=_status_text) + svc.add_path('/ErrorMessage', '') + svc.add_path('/IsStationary', 0) + svc.add_path('/Depth/Current', None) + svc.add_path('/Depth/History', '') + svc.add_path('/Depth/FullHistory', '') + svc.add_path('/Tide/Predictions', '') + svc.add_path('/Tide/Station/Predictions', '') + svc.add_path('/Tide/Local/Predictions', '') + svc.add_path('/Tide/Local/TimeOffset', 0.0) + svc.add_path('/Tide/Local/AmpScale', 1.0) + svc.add_path('/Tide/Local/MatchCount', 0) + svc.add_path('/Tide/Local/Status', 0) + for slot in ('NextHigh1', 'NextHigh2', 'NextLow1', 'NextLow2', + 'PrevHigh1', 'PrevHigh2', 'PrevLow1', 'PrevLow2'): + svc.add_path(f'/Tide/{slot}/Time', None) + svc.add_path(f'/Tide/{slot}/Depth', None) + svc.add_path(f'/Tide/{slot}/ObsTime', None) + svc.add_path(f'/Tide/{slot}/ObsDepth', None) + svc.add_path('/Tide/LastModelRun', 0) + svc.add_path('/Tide/DataSource', '') + svc.add_path('/Tide/Station/Id', '') + svc.add_path('/Tide/Station/Name', '') + svc.add_path('/Tide/Station/Distance', None) + svc.add_path('/Tide/Station/Lat', None) + svc.add_path('/Tide/Station/Lon', None) + svc.add_path('/Tide/Station/Type', '') + svc.add_path('/Tide/Station/RefId', '') + svc.add_path('/Tide/NearbyStations', '[]') + svc.add_path('/Tide/ChartDepth', 0.0) + svc.add_path('/Tide/DatumOffset', 0.0) + svc.add_path('/Tide/ModelLocation/Lat', None) + svc.add_path('/Tide/ModelLocation/Lon', None) + svc.add_path('/Settings/Enabled', 1, + writeable=True, + onchangecallback=self._on_setting_changed) + svc.add_path('/Settings/MinTideAmplitude', 0.3, + writeable=True, + onchangecallback=self._on_setting_changed) + svc.add_path('/Settings/Units', 0, + writeable=True, + onchangecallback=self._on_setting_changed) + svc.add_path('/Settings/DatumOffset', -1.0, + writeable=True, + onchangecallback=self._on_setting_changed) + svc.add_path('/Settings/StationOverride', '', + writeable=True, + onchangecallback=self._on_setting_changed) + + svc.register() + self.logger.info("D-Bus service created") + + def _setup_settings(self): + self.settings = None + self.enabled = True + try: + path = '/Settings/Tides' + settings_def = { + 'Enabled': [path + '/Enabled', 1, 0, 1], + 'Units': [path + '/Units', 0, 0, 1], + } + self.settings = SettingsDevice( + self.bus, settings_def, + self._on_persistent_setting_changed) + if self.settings: + self.enabled = bool(self.settings['Enabled']) + self.dbus_service['/Settings/Enabled'] = ( + 1 if self.enabled else 0) + self.dbus_service['/Settings/Units'] = ( + int(self.settings['Units'])) + self.logger.info("Persistent settings initialized") + except Exception as e: + self.logger.warning("Could not init persistent settings: %s", e) + + def _on_persistent_setting_changed(self, setting, old_value, new_value): + self.logger.info("Setting changed: %s = %s", setting, new_value) + if self.settings: + self.enabled = bool(self.settings['Enabled']) + + def _on_setting_changed(self, path, value): + self.logger.info("Setting changed: %s = %s", path, value) + if path == '/Settings/Enabled': + self.enabled = bool(value) + if self.settings: + try: + self.settings['Enabled'] = 1 if self.enabled else 0 + except Exception: + pass + elif path == '/Settings/MinTideAmplitude': + try: + self.tide_detector.min_amplitude = float(value) + except (TypeError, ValueError): + pass + elif path == '/Settings/Units': + if self.settings: + try: + self.settings['Units'] = int(value) + except Exception: + pass + elif path == '/Settings/StationOverride': + self.tide_predictor.station_override = str(value) if value else None + self.last_prediction_run = 0 + return True + + def _main_loop(self): + try: + if not self.enabled: + self.dbus_service['/Status'] = 0 + return True + + now = time.time() + self._check_gps(now) + self._check_depth(now) + self._check_predictions(now) + self._refresh_event_slots(now) + + except dbus.exceptions.DBusException as e: + self.logger.warning("D-Bus error: %s", e) + except Exception as e: + self.logger.exception("Unexpected error: %s", e) + + return True + + def _check_gps(self, now): + if now - self.last_gps_check < GPS_SAMPLE_INTERVAL: + return + + pos = self.gps.get_position() + if pos: + self.current_lat, self.current_lon = pos + + if (now - self.last_gps_history_sample + >= GPS_HISTORY_SAMPLE_INTERVAL): + self.movement_stationary.add_position( + self.current_lat, self.current_lon, now) + self.movement_model.add_position( + self.current_lat, self.current_lon, now) + self.last_gps_history_sample = now + + is_stationary = not self.movement_stationary.is_moving( + self.current_lat, self.current_lon) + self.dbus_service['/IsStationary'] = 1 if is_stationary else 0 + + self.last_gps_check = now + + def _check_depth(self, now): + if now - self.last_depth_sample < DEPTH_SAMPLE_INTERVAL: + return + + speed = self.gps.get_speed() + if speed is not None and speed > SPEED_THRESHOLD_MS: + self.last_depth_sample = now + return + + depth = self.depth_reader.get_depth() + if depth is None: + self.last_depth_sample = now + return + + result = self.depth_recorder.add_reading( + depth, self.current_lat, self.current_lon, now) + + smoothed = self.depth_recorder.get_latest_depth() + if smoothed is not None: + self.dbus_service['/Depth/Current'] = round(smoothed, 2) + + if result is not None: + history = self.depth_recorder.get_history( + lat=self.current_lat, lon=self.current_lon) + new_event = self.tide_detector.update(history) + + self._recalibrate_if_moved(history) + + now_ts = time.time() + display_cutoff = now_ts - 24 * 3600 + obs_points = [ + {'ts': int(ts), 'depth': round(d, 2)} + for ts, d in history if ts > display_cutoff + ] + self.dbus_service['/Depth/History'] = json.dumps(obs_points) + + full_obs = [ + {'ts': int(ts), 'depth': round(d, 2)} + for ts, d in history + ] + self.dbus_service['/Depth/FullHistory'] = json.dumps(full_obs) + + if new_event is not None: + self._refresh_adaptation() + + self.last_depth_sample = now + + def _recalibrate_if_moved(self, history): + """Recalibrate chart_depth if vessel has moved to a new position.""" + if not self.tide_merger.needs_recalibration( + self.current_lat, self.current_lon): + return + pred_ts, pred_heights, _ = ( + self.tide_predictor.get_last_prediction()) + if pred_ts is None: + return + if self.tide_merger.calibrate( + history, pred_ts, pred_heights, + lat=self.current_lat, lon=self.current_lon): + self._publish_adaptation(pred_ts, pred_heights, + self.tide_predictor._last_extrema) + + def _refresh_adaptation(self): + """Recompute adaptation from observed events and last prediction.""" + pred_ts, pred_heights, pred_extrema = ( + self.tide_predictor.get_last_prediction()) + if pred_extrema is None: + return + + observed_events = self.tide_detector.get_recent_events(20) + if not observed_events: + return + + if self.tide_adapter.update(observed_events, pred_extrema): + self._publish_adaptation(pred_ts, pred_heights, pred_extrema) + + def _refresh_event_slots(self, now): + """Re-classify next/prev tide events every 60s using cached extrema. + + Keeps the dbus slots current as predicted event times pass, + without waiting for a new prediction run or observed event. + """ + if now - self.last_event_refresh < 60: + return + + self.last_event_refresh = now + + pred_ts, pred_heights, extrema = ( + self.tide_predictor.get_last_prediction()) + if extrema is None: + return + + chart_depth = self.tide_merger.chart_depth or 0.0 + + best_extrema = extrema + if self.tide_adapter.status > 0: + best_extrema = self.tide_adapter.adapt_extrema(extrema) + + calibrated_extrema = [ + (ts, chart_depth + h, t) for ts, h, t in best_extrema] + next_tides = self.tide_merger.get_next_tides( + calibrated_extrema, count=2) + prev_tides = self.tide_merger.get_prev_tides( + calibrated_extrema, count=2) + observed_events = self.tide_detector.get_recent_events(20) + self._publish_tide_events(next_tides, prev_tides, observed_events) + + def _check_predictions(self, now): + if self._predict_in_progress: + return + if self.current_lat is None or self.current_lon is None: + return + + needs_model = False + + if self.model_lat is None: + needs_model = True + elif now - self.last_prediction_run >= MODEL_RERUN_INTERVAL: + needs_model = True + elif self.movement_model.is_moving( + self.current_lat, self.current_lon): + needs_model = True + + if needs_model: + self._predict_in_progress = True + thread = threading.Thread( + target=self._run_prediction, + args=(self.current_lat, self.current_lon, now), + daemon=True) + thread.start() + + def _run_prediction(self, lat, lon, now): + """Run in background thread.""" + try: + self.logger.info( + "Running tide prediction for %.2f, %.2f", lat, lon) + self.dbus_service['/Status'] = 1 + + history = self.depth_recorder.get_history(lat=lat, lon=lon) + + if not self.tide_predictor.update_location( + lat, lon, depth_history=history): + self.dbus_service['/Status'] = 3 + self.dbus_service['/ErrorMessage'] = 'No constituent data' + return + + pred_ts, pred_heights, extrema = ( + self.tide_predictor.predict( + now - DEPTH_HISTORY_HOURS * 3600, + duration_hours=DEPTH_HISTORY_HOURS + PREDICTION_HOURS)) + if pred_ts is None: + self.dbus_service['/Status'] = 3 + self.dbus_service['/ErrorMessage'] = 'Prediction failed' + return + + if history: + self.tide_merger.calibrate( + history, pred_ts, pred_heights, lat=lat, lon=lon) + + if history: + self.tide_detector.update( + [(ts, d) for ts, d in history]) + observed_events = self.tide_detector.get_recent_events(20) + if observed_events and extrema: + self.tide_adapter.update(observed_events, extrema) + + self._publish_adaptation(pred_ts, pred_heights, extrema) + + manual_offset = self.dbus_service['/Settings/DatumOffset'] + if manual_offset is not None and manual_offset >= 0: + datum_offset = float(manual_offset) + else: + datum_offset = self.tide_predictor.estimate_datum_offset() + self.dbus_service['/Tide/DatumOffset'] = round(datum_offset, 3) + + self.model_lat = lat + self.model_lon = lon + self.last_prediction_run = time.time() + + self.dbus_service['/Tide/LastModelRun'] = int( + self.last_prediction_run) + self.dbus_service['/Tide/DataSource'] = ( + self.tide_predictor.current_source or '') + self._publish_station_info() + self.dbus_service['/Tide/ModelLocation/Lat'] = lat + self.dbus_service['/Tide/ModelLocation/Lon'] = lon + self.dbus_service['/ErrorMessage'] = '' + self.dbus_service['/Status'] = 2 + + self.logger.info( + "Prediction complete, %d extrema found", + len(extrema) if extrema else 0) + + except Exception as e: + self.logger.exception("Prediction error: %s", e) + self.dbus_service['/Status'] = 3 + self.dbus_service['/ErrorMessage'] = str(e)[:200] + finally: + self._predict_in_progress = False + + def _publish_station_info(self): + """Publish selected station metadata and nearby stations list.""" + svc = self.dbus_service + st = self.tide_predictor.current_station + if st: + svc['/Tide/Station/Id'] = st.get('id', '') + svc['/Tide/Station/Name'] = st.get('name', '') + svc['/Tide/Station/Distance'] = round( + st.get('distance', 0) / 1000, 1) + svc['/Tide/Station/Lat'] = st.get('lat') + svc['/Tide/Station/Lon'] = st.get('lon') + svc['/Tide/Station/Type'] = st.get('type', 'R') + svc['/Tide/Station/RefId'] = st.get('ref_id', '') + else: + svc['/Tide/Station/Id'] = '' + svc['/Tide/Station/Name'] = '' + svc['/Tide/Station/Distance'] = None + svc['/Tide/Station/Lat'] = None + svc['/Tide/Station/Lon'] = None + svc['/Tide/Station/Type'] = '' + svc['/Tide/Station/RefId'] = '' + + nearby = self.tide_predictor.nearby_stations + svc['/Tide/NearbyStations'] = json.dumps(nearby) + + def _publish_adaptation(self, pred_ts, pred_heights, extrema): + """Build and publish station + local curves and next-tide data.""" + history = self.depth_recorder.get_history( + lat=self.current_lat, lon=self.current_lon) + chart_depth = self.tide_merger.chart_depth or 0.0 + + self.dbus_service['/Tide/ChartDepth'] = round(chart_depth, 3) + station_depths = [chart_depth + h for h in pred_heights] + + local_ts = None + local_depths = None + best_extrema = extrema + + if self.tide_adapter.status > 0: + adapted_ts, adapted_h = self.tide_adapter.adapt_curve( + pred_ts, pred_heights) + local_depths = [chart_depth + h for h in adapted_h] + local_ts = adapted_ts + best_extrema = self.tide_adapter.adapt_extrema(extrema) + + residual = self.tide_adapter.compute_residual_correction( + history, self.tide_predictor.predict_at, chart_depth) + if residual: + local_depths = self.tide_adapter.apply_residual_correction( + local_ts, local_depths, residual) + + curves = self.tide_merger.build_dual_curves( + history, pred_ts, station_depths, local_ts, local_depths) + + self.dbus_service['/Depth/History'] = json.dumps( + curves.get('observed', [])) + + full_obs = [ + {'ts': int(ts), 'depth': round(d, 2)} + for ts, d in history + ] + self.dbus_service['/Depth/FullHistory'] = json.dumps(full_obs) + self.dbus_service['/Tide/Station/Predictions'] = json.dumps( + curves.get('station', [])) + self.dbus_service['/Tide/Local/Predictions'] = json.dumps( + curves.get('local', [])) + + best_curve = curves.get('local', []) or curves.get('station', []) + self.dbus_service['/Tide/Predictions'] = json.dumps(best_curve) + + self.dbus_service['/Tide/Local/TimeOffset'] = round( + self.tide_adapter.time_offset, 1) + self.dbus_service['/Tide/Local/AmpScale'] = round( + self.tide_adapter.amp_scale, 3) + self.dbus_service['/Tide/Local/MatchCount'] = ( + self.tide_adapter.match_count) + self.dbus_service['/Tide/Local/Status'] = self.tide_adapter.status + + if best_extrema: + calibrated_extrema = [ + (ts, chart_depth + h, t) for ts, h, t in best_extrema] + next_tides = self.tide_merger.get_next_tides( + calibrated_extrema, count=2) + prev_tides = self.tide_merger.get_prev_tides( + calibrated_extrema, count=2) + observed_events = self.tide_detector.get_recent_events(20) + self._publish_tide_events( + next_tides, prev_tides, observed_events) + + def _publish_tide_events(self, next_tides, prev_tides, + observed_events=None): + svc = self.dbus_service + + obs_by_type = {'high': [], 'low': []} + if observed_events: + for obs_ts, obs_depth, obs_type in observed_events: + obs_by_type.setdefault(obs_type, []).append( + (obs_ts, obs_depth)) + + def _find_obs(pred_ts, tide_type, max_window=10800): + """Find the observed event closest to a predicted timestamp.""" + best, best_dist = None, max_window + for obs_ts, obs_depth in obs_by_type.get(tide_type, []): + dist = abs(obs_ts - pred_ts) + if dist < best_dist: + best_dist = dist + best = (obs_ts, obs_depth) + return best + + def _set_slot(slot, pred_list, idx, tide_type): + if idx < len(pred_list): + ts, depth = pred_list[idx] + svc[f'/Tide/{slot}/Time'] = int(ts) + svc[f'/Tide/{slot}/Depth'] = round(depth, 2) + obs = _find_obs(ts, tide_type) + if obs: + svc[f'/Tide/{slot}/ObsTime'] = int(obs[0]) + svc[f'/Tide/{slot}/ObsDepth'] = round(obs[1], 2) + else: + svc[f'/Tide/{slot}/ObsTime'] = None + svc[f'/Tide/{slot}/ObsDepth'] = None + else: + for suffix in ('Time', 'Depth', 'ObsTime', 'ObsDepth'): + svc[f'/Tide/{slot}/{suffix}'] = None + + highs = next_tides.get('high', []) + lows = next_tides.get('low', []) + _set_slot('NextHigh1', highs, 0, 'high') + _set_slot('NextHigh2', highs, 1, 'high') + _set_slot('NextLow1', lows, 0, 'low') + _set_slot('NextLow2', lows, 1, 'low') + + prev_highs = prev_tides.get('high', []) + prev_lows = prev_tides.get('low', []) + _set_slot('PrevHigh1', prev_highs, 0, 'high') + _set_slot('PrevHigh2', prev_highs, 1, 'high') + _set_slot('PrevLow1', prev_lows, 0, 'low') + _set_slot('PrevLow2', prev_lows, 1, 'low') + + +def main(): + DBusGMainLoop(set_as_default=True) + + print("=" * 60) + print("dbus-tides v%s" % VERSION) + print("=" * 60) + + mainloop = None + + def signal_handler(signum, frame): + try: + sig_name = signal.Signals(signum).name + except ValueError: + sig_name = str(signum) + logging.info("Received %s, shutting down...", sig_name) + if mainloop is not None: + mainloop.quit() + + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + + try: + controller = TidesController() + mainloop = GLib.MainLoop() + mainloop.run() + except KeyboardInterrupt: + print("\nShutdown requested") + except Exception as e: + logging.error("Fatal error: %s", e, exc_info=True) + sys.exit(1) + finally: + logging.info("Service stopped") + + +if __name__ == '__main__': + main() diff --git a/dbus-tides/tools/build_coastal_grid.py b/dbus-tides/tools/build_coastal_grid.py new file mode 100644 index 0000000..0191f63 --- /dev/null +++ b/dbus-tides/tools/build_coastal_grid.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +""" +Build the pre-computed coastal tidal constituent grid. + +This script runs on a DEVELOPMENT MACHINE where pyTMD is installed. +It extracts tidal constituents at every 0.25-degree ocean point across +the configured regions, masks land, and writes a gzipped JSON file. + +Usage: + pip install -r requirements-dev.txt + python build_coastal_grid.py [--resolution 0.25] [--model GOT4.10_nc] + +Output: ../constituents/coastal_grid.json.gz +""" + +import argparse +import gzip +import json +import os +import sys +import time + +try: + import numpy as np + import pyTMD +except ImportError: + print("ERROR: pyTMD is not installed. Run: pip install -r requirements-dev.txt") + sys.exit(1) + +REGIONS = [ + ('us_east', 24.0, 45.0, -82.0, -66.0), + ('bahamas', 20.0, 28.0, -80.0, -72.0), + ('caribbean', 10.0, 24.0, -88.0, -59.0), + ('gulf', 24.0, 31.0, -98.0, -82.0), + ('us_west', 32.0, 49.0, -125.0, -117.0), +] + + +def main(): + parser = argparse.ArgumentParser( + description='Build coastal tidal constituent grid') + parser.add_argument('--resolution', type=float, default=0.25, + help='Grid resolution in degrees (default: 0.25)') + parser.add_argument('--model', type=str, default='GOT4.10_nc', + help='pyTMD database model name (default: GOT4.10_nc)') + parser.add_argument('--output', type=str, + default='../constituents/coastal_grid.json.gz', + help='Output file path') + args = parser.parse_args() + + res = args.resolution + print(f"Building coastal grid at {res}-degree resolution") + print(f"Model: {args.model}") + print(f"Regions: {[r[0] for r in REGIONS]}") + print() + + all_coords = set() + for name, lat_min, lat_max, lon_min, lon_max in REGIONS: + lat = lat_min + while lat <= lat_max + 1e-9: + lon = lon_min + while lon <= lon_max + 1e-9: + all_coords.add((round(lat, 4), round(lon, 4))) + lon += res + lat += res + print(f" {name}: {lat_min}N-{lat_max}N, " + f"{abs(lon_max)}W-{abs(lon_min)}W") + + coords = sorted(all_coords) + print(f"\nTotal grid points to extract: {len(coords)}") + + lats = np.array([c[0] for c in coords]) + lons_neg = np.array([c[1] for c in coords]) + lons_360 = lons_neg % 360 + + print("Loading tidal model...") + t0 = time.time() + + try: + model = pyTMD.io.model().from_database(args.model) + except Exception as e: + print(f"ERROR loading model from database: {e}", file=sys.stderr) + sys.exit(1) + + try: + ds = pyTMD.io.GOT.open_mfdataset(model.z.model_file) + except Exception as e: + print(f"ERROR opening model files: {e}", file=sys.stderr) + sys.exit(1) + + const_names = list(ds.data_vars) + print(f"Constituents in model: {const_names}") + + unique_lats = np.unique(lats) + unique_lons = np.unique(lons_360) + + print(f"Interpolating on {len(unique_lats)} unique lats x " + f"{len(unique_lons)} unique lons...") + + ds_interp = ds.interp(y=unique_lats, x=unique_lons) + + elapsed = time.time() - t0 + print(f"Interpolation complete in {elapsed:.1f}s") + + lat_to_idx = {float(v): i for i, v in enumerate(unique_lats)} + lon_to_idx = {float(v): i for i, v in enumerate(unique_lons)} + + points = [] + n_land = 0 + for i in range(len(coords)): + lat_i = lat_to_idx[float(lats[i])] + lon_i = lon_to_idx[float(lons_360[i])] + + amplitudes = [] + phases = [] + all_nan = True + + for cname in const_names: + val = complex(ds_interp[cname].values[lat_i, lon_i]) + if np.isnan(val): + amplitudes.append(0.0) + phases.append(0.0) + else: + all_nan = False + amp_cm = np.abs(val) + amp_m = amp_cm / 100.0 + phase_deg = (-np.angle(val, deg=True)) % 360 + amplitudes.append(round(float(amp_m), 6)) + phases.append(round(float(phase_deg), 4)) + + if all_nan or max(amplitudes) < 1e-7: + n_land += 1 + continue + + points.append({ + 'lat': float(lats[i]), + 'lon': float(lons_neg[i]), + 'amp': amplitudes, + 'phase': phases, + }) + + print(f"\nOcean points: {len(points)}") + print(f"Land points (masked): {n_land}") + + grid_data = { + 'regions': [r[0] for r in REGIONS], + 'resolution_deg': res, + 'constituents': [str(n).upper() for n in const_names], + 'points': points, + } + + print(f"\nWriting {args.output}...") + json_str = json.dumps(grid_data, separators=(',', ':')) + + os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) + + if args.output.endswith('.gz'): + with gzip.open(args.output, 'wt', encoding='utf-8') as f: + f.write(json_str) + else: + with open(args.output, 'w') as f: + f.write(json_str) + + size = os.path.getsize(args.output) + print(f"Done: {size / 1024:.0f} KB compressed") + print(f"Uncompressed JSON: {len(json_str) / 1024:.0f} KB") + + +if __name__ == '__main__': + main() diff --git a/dbus-tides/tools/build_noaa_stations.py b/dbus-tides/tools/build_noaa_stations.py new file mode 100644 index 0000000..c7348bc --- /dev/null +++ b/dbus-tides/tools/build_noaa_stations.py @@ -0,0 +1,394 @@ +#!/usr/bin/env python3 +""" +Build a pre-computed NOAA tide station database. + +Fetches harmonic constituents from the NOAA CO-OPS Metadata API for all +tide prediction stations within the configured geographic regions. +Outputs a gzipped JSON file that the runtime predictor uses as its +highest-priority constituent source (observation-derived data is more +accurate near coastlines than satellite-altimetry models like GOT4.10). + +Usage: + python build_noaa_stations.py + python build_noaa_stations.py --output ../constituents/noaa_stations.json.gz + python build_noaa_stations.py --regions us_east,bahamas,caribbean + +Output: ../constituents/noaa_stations.json.gz +""" + +import argparse +import gzip +import http.client +import json +import os +import sys +import time + +try: + from urllib.request import urlopen, Request + from urllib.error import URLError, HTTPError +except ImportError: + print("ERROR: urllib not available", file=sys.stderr) + sys.exit(1) + +NOAA_BASE = 'https://api.tidesandcurrents.noaa.gov/mdapi/prod/webapi' + +REGIONS = { + 'us_east': (24.0, 45.0, -82.0, -66.0), + 'bahamas': (20.0, 28.0, -80.0, -72.0), + 'caribbean': (10.0, 24.0, -88.0, -59.0), + 'gulf': (24.0, 31.0, -98.0, -82.0), + 'us_west': (32.0, 49.0, -125.0, -117.0), +} + +# NOAA constituent names that differ from the dbus-tides naming convention +NOAA_NAME_MAP = { + 'RHO': 'RHO1', + 'MSF': 'MS', +} + + +def _read_response(resp): + """Read an HTTP response body, recovering partial data on truncation.""" + expected = resp.getheader('Content-Length') + expected = int(expected) if expected else None + chunks = [] + total = 0 + try: + while True: + chunk = resp.read(65536) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + except http.client.IncompleteRead as e: + chunks.append(e.partial) + total += len(e.partial) + body = b''.join(chunks) + if expected and total < expected: + raise http.client.IncompleteRead(body, expected - total) + return body + + +def fetch_json(url, retries=5, delay=3.0, timeout=120): + """Fetch JSON from a URL with retries and chunked reading.""" + headers = { + 'Accept': 'application/json', + 'Accept-Encoding': 'identity', + 'Connection': 'keep-alive', + } + last_err = None + body = b'' + for attempt in range(retries): + try: + req = Request(url, headers=headers) + with urlopen(req, timeout=timeout) as resp: + body = _read_response(resp) + return json.loads(body.decode('utf-8')) + except json.JSONDecodeError as e: + last_err = e + if attempt < retries - 1: + wait = delay * (attempt + 1) + print(f" Retry {attempt+1}/{retries}: truncated response " + f"({len(body)} bytes) from {url}", + file=sys.stderr) + time.sleep(wait) + continue + except (URLError, HTTPError, TimeoutError, + http.client.IncompleteRead, + ConnectionResetError, OSError) as e: + last_err = e + if attempt < retries - 1: + wait = delay * (attempt + 1) + print(f" Retry {attempt+1}/{retries} for {url}: {e}", + file=sys.stderr) + time.sleep(wait) + continue + raise RuntimeError(f"Failed to fetch {url}: {last_err}") + + +def get_stations(): + """Fetch the full list of NOAA tide prediction stations.""" + url = f'{NOAA_BASE}/stations.json?type=tidepredictions&units=metric' + data = fetch_json(url) + stations = data.get('stations', []) + return stations + + +def in_region(lat, lon, bounds): + """Check if a point falls within a lat/lon bounding box.""" + lat_min, lat_max, lon_min, lon_max = bounds + return lat_min <= lat <= lat_max and lon_min <= lon <= lon_max + + +def in_any_region(lat, lon, region_bounds): + """Check if a point falls within any of the configured regions.""" + for bounds in region_bounds: + if in_region(lat, lon, bounds): + return True + return False + + +def get_harcon(station_id): + """Fetch harmonic constituents for a single station.""" + url = (f'{NOAA_BASE}/stations/{station_id}/harcon.json' + f'?units=metric') + data = fetch_json(url) + return data.get('HarmonicConstituents', []) + + +def get_tidepredoffsets(station_id): + """Fetch subordinate station tide prediction offsets.""" + url = (f'{NOAA_BASE}/stations/{station_id}' + f'/tidepredoffsets.json') + data = fetch_json(url) + return data + + +def map_constituent_name(noaa_name): + """Map a NOAA constituent name to the dbus-tides convention.""" + upper = noaa_name.strip().upper() + return NOAA_NAME_MAP.get(upper, upper) + + +def main(): + parser = argparse.ArgumentParser( + description='Build NOAA tide station database') + parser.add_argument( + '--output', type=str, + default='../constituents/noaa_stations.json.gz', + help='Output file path (default: ../constituents/noaa_stations.json.gz)') + parser.add_argument( + '--regions', type=str, + default='us_east,bahamas,caribbean', + help='Comma-separated region names (default: us_east,bahamas,caribbean)') + parser.add_argument( + '--all-regions', action='store_true', + help='Include all available regions') + parser.add_argument( + '--delay', type=float, default=0.25, + help='Delay between API requests in seconds (default: 0.25)') + args = parser.parse_args() + + if args.all_regions: + region_names = list(REGIONS.keys()) + else: + region_names = [r.strip() for r in args.regions.split(',')] + + region_bounds = [] + for name in region_names: + if name not in REGIONS: + print(f"ERROR: Unknown region '{name}'. " + f"Available: {', '.join(REGIONS.keys())}", + file=sys.stderr) + sys.exit(1) + region_bounds.append(REGIONS[name]) + + print(f"Regions: {region_names}") + print(f"Fetching NOAA station list...") + + all_stations = get_stations() + print(f"Total NOAA tide prediction stations: {len(all_stations)}") + + filtered = [] + for st in all_stations: + lat = st.get('lat') + lng = st.get('lng') + if lat is None or lng is None: + continue + try: + lat = float(lat) + lng = float(lng) + except (ValueError, TypeError): + continue + if in_any_region(lat, lng, region_bounds): + filtered.append((st, lat, lng)) + + harmonic_filtered = [(st, lat, lng) + for st, lat, lng in filtered + if st.get('type') == 'R'] + subordinate_filtered = [(st, lat, lng) + for st, lat, lng in filtered + if st.get('type') == 'S'] + print(f"Stations in selected regions: {len(filtered)}") + print(f" Harmonic (R): {len(harmonic_filtered)}") + print(f" Subordinate (S): {len(subordinate_filtered)}") + + if not filtered: + print("No stations found in the selected regions.", file=sys.stderr) + sys.exit(1) + + # --- Phase 1: Fetch harmonic stations --- + stations_out = [] + errors = 0 + print(f"\nPhase 1: Fetching {len(harmonic_filtered)} harmonic stations...") + for i, (st, lat, lng) in enumerate(harmonic_filtered): + station_id = st.get('id', '') + station_name = st.get('name', '') + + if (i + 1) % 10 == 0 or i == 0: + print(f" [{i+1}/{len(harmonic_filtered)}] {station_id} " + f"{station_name}...") + + try: + harcon = get_harcon(station_id) + except RuntimeError as e: + print(f" WARNING: {e}", file=sys.stderr) + errors += 1 + continue + + if not harcon: + continue + + constituents = [] + for hc in harcon: + name = map_constituent_name(hc.get('name', '')) + amp = hc.get('amplitude', 0.0) + phase = hc.get('phase_GMT', 0.0) + if amp and amp > 0: + constituents.append({ + 'name': name, + 'amplitude': round(float(amp), 6), + 'phase': round(float(phase), 4), + }) + + if constituents: + stations_out.append({ + 'id': station_id, + 'name': station_name, + 'lat': lat, + 'lon': lng, + 'constituents': constituents, + }) + + if args.delay > 0 and i < len(harmonic_filtered) - 1: + time.sleep(args.delay) + + harmonic_ids = {s['id'] for s in stations_out} + print(f" Harmonic stations with data: {len(stations_out)}") + + # --- Phase 2: Fetch subordinate stations --- + sub_count = 0 + sub_errors = 0 + print(f"\nPhase 2: Fetching {len(subordinate_filtered)} " + f"subordinate stations...") + for i, (st, lat, lng) in enumerate(subordinate_filtered): + station_id = st.get('id', '') + station_name = st.get('name', '') + ref_id = st.get('reference_id', '') + + if (i + 1) % 50 == 0 or i == 0: + print(f" [{i+1}/{len(subordinate_filtered)}] {station_id} " + f"{station_name} (ref: {ref_id})...") + + if not ref_id: + continue + + try: + offsets = get_tidepredoffsets(station_id) + except RuntimeError as e: + print(f" WARNING: {e}", file=sys.stderr) + sub_errors += 1 + continue + + if not offsets or not offsets.get('refStationId'): + continue + + ref_station_id = offsets['refStationId'] + + # Ensure the reference station is in our harmonic set; if not + # and it's a valid ID, fetch its constituents too. + if ref_station_id not in harmonic_ids: + try: + harcon = get_harcon(ref_station_id) + except RuntimeError: + harcon = [] + if harcon: + ref_st = next( + (s for s in all_stations + if s.get('id') == ref_station_id), None) + ref_lat = float(ref_st['lat']) if ref_st else 0.0 + ref_lon = float(ref_st.get('lng', 0)) if ref_st else 0.0 + ref_name = ref_st.get('name', '') if ref_st else '' + constituents = [] + for hc in harcon: + name = map_constituent_name(hc.get('name', '')) + amp = hc.get('amplitude', 0.0) + phase = hc.get('phase_GMT', 0.0) + if amp and amp > 0: + constituents.append({ + 'name': name, + 'amplitude': round(float(amp), 6), + 'phase': round(float(phase), 4), + }) + if constituents: + stations_out.append({ + 'id': ref_station_id, + 'name': ref_name, + 'lat': ref_lat, + 'lon': ref_lon, + 'constituents': constituents, + }) + harmonic_ids.add(ref_station_id) + print(f" Added reference station {ref_station_id} " + f"{ref_name}") + + if ref_station_id not in harmonic_ids: + continue + + if args.delay > 0: + time.sleep(args.delay) + + height_type = offsets.get('heightAdjustedType', 'R') + stations_out.append({ + 'id': station_id, + 'name': station_name, + 'lat': lat, + 'lon': lng, + 'type': 'S', + 'ref_id': ref_station_id, + 'offsets': { + 'time_high': offsets.get('timeOffsetHighTide', 0), + 'time_low': offsets.get('timeOffsetLowTide', 0), + 'height_high': offsets.get('heightOffsetHighTide', 1.0), + 'height_low': offsets.get('heightOffsetLowTide', 1.0), + 'height_type': height_type, + }, + }) + sub_count += 1 + + if args.delay > 0 and i < len(subordinate_filtered) - 1: + time.sleep(args.delay) + + print(f" Subordinate stations added: {sub_count}") + if errors or sub_errors: + print(f" Errors: {errors} harmonic, {sub_errors} subordinate") + + print(f"\nTotal stations: {len(stations_out)}") + + output_data = { + 'source': 'NOAA CO-OPS', + 'regions': region_names, + 'fetched': time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime()), + 'stations': stations_out, + } + + print(f"\nWriting {args.output}...") + json_str = json.dumps(output_data, separators=(',', ':')) + + os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) + + if args.output.endswith('.gz'): + with gzip.open(args.output, 'wt', encoding='utf-8') as f: + f.write(json_str) + else: + with open(args.output, 'w') as f: + f.write(json_str) + + size = os.path.getsize(args.output) + print(f"Done: {size / 1024:.0f} KB compressed") + print(f"Uncompressed JSON: {len(json_str) / 1024:.0f} KB") + + +if __name__ == '__main__': + main() diff --git a/dbus-tides/tools/extract_constituents.py b/dbus-tides/tools/extract_constituents.py new file mode 100644 index 0000000..4f2b294 --- /dev/null +++ b/dbus-tides/tools/extract_constituents.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +""" +Extract tidal constituents for a single lat/lon from a tidal model. + +This script runs on a DEVELOPMENT MACHINE (not the Cerbo) where pyTMD +and its dependencies (numpy, scipy, etc.) are available via pip. + +Usage: + pip install -r requirements-dev.txt + python extract_constituents.py --lat 25.0 --lon -77.5 --output ../constituents/nassau.json + +The output JSON can be bundled with the dbus-tides package for use on the Cerbo. +""" + +import argparse +import json +import sys + +try: + import pyTMD + from pyTMD.io.model import load_constituents +except ImportError: + print("ERROR: pyTMD is not installed. Run: pip install -r requirements-dev.txt") + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser( + description='Extract tidal constituents for a location') + parser.add_argument('--lat', type=float, required=True, + help='Latitude in decimal degrees') + parser.add_argument('--lon', type=float, required=True, + help='Longitude in decimal degrees') + parser.add_argument('--model', type=str, default='GOT4.10c', + help='Tidal model name (default: GOT4.10c)') + parser.add_argument('--model-dir', type=str, default=None, + help='Directory containing model files') + parser.add_argument('--output', type=str, required=True, + help='Output JSON file path') + args = parser.parse_args() + + print(f"Extracting constituents for ({args.lat}, {args.lon}) " + f"using model {args.model}...") + + try: + import numpy as np + + model = pyTMD.io.model(directory=args.model_dir).elevation(args.model) + + amp, ph, c = pyTMD.io.extract_constants( + np.array([args.lon]), + np.array([args.lat]), + model + ) + + constituents = [] + for i, name in enumerate(c): + a = float(amp[0, i]) + p = float(ph[0, i]) + if a > 0: + constituents.append({ + 'name': name.upper(), + 'amplitude': round(a, 6), + 'phase': round(p, 4), + }) + + output = { + 'location': { + 'lat': args.lat, + 'lon': args.lon, + }, + 'model': args.model, + 'constituents': constituents, + } + + with open(args.output, 'w') as f: + json.dump(output, f, indent=2) + + print(f"Wrote {len(constituents)} constituents to {args.output}") + + except Exception as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/dbus-tides/tools/noaa_stations-bahamas.json.gz b/dbus-tides/tools/noaa_stations-bahamas.json.gz new file mode 100644 index 0000000..cd71f6c Binary files /dev/null and b/dbus-tides/tools/noaa_stations-bahamas.json.gz differ diff --git a/dbus-tides/tools/predict_tides.py b/dbus-tides/tools/predict_tides.py new file mode 100644 index 0000000..85bbdca --- /dev/null +++ b/dbus-tides/tools/predict_tides.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +""" +Standalone tide prediction for a given GPS location using the coastal grid. + +Usage: + python tools/predict_tides.py [--lat 25.49] [--lon -76.64] +""" + +import argparse +import gzip +import json +import math +import os +import sys +import time +from datetime import datetime, timezone + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import tide_harmonics + + +def haversine(lat1, lon1, lat2, lon2): + R = 6371000 + phi1, phi2 = math.radians(lat1), math.radians(lat2) + dphi = math.radians(lat2 - lat1) + dlam = math.radians(lon2 - lon1) + a = (math.sin(dphi / 2) ** 2 + + math.cos(phi1) * math.cos(phi2) * math.sin(dlam / 2) ** 2) + return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) + + +def load_grid(grid_path): + if grid_path.endswith('.gz'): + with gzip.open(grid_path, 'rt', encoding='utf-8') as f: + return json.load(f) + with open(grid_path, 'r') as f: + return json.load(f) + + +def interpolate_constituents(grid, lat, lon): + points = grid['points'] + const_names = grid['constituents'] + + dists = [] + for pt in points: + d = haversine(lat, lon, pt['lat'], pt['lon']) + dists.append((d, pt)) + dists.sort(key=lambda x: x[0]) + nearest = dists[:4] + + closest_dist, closest_pt = nearest[0] + print(f"Nearest grid point: ({closest_pt['lat']}, {closest_pt['lon']}) " + f"at {closest_dist / 1000:.1f} km") + + if closest_dist < 1000 or len(nearest) < 4: + amps = closest_pt.get('amp', []) + phases = closest_pt.get('phase', []) + return [{'name': const_names[j], 'amplitude': amps[j], 'phase': phases[j]} + for j in range(min(len(const_names), len(amps), len(phases)))] + + total_weight = 0.0 + n = len(const_names) + weighted_amp = [0.0] * n + weighted_phase_cos = [0.0] * n + weighted_phase_sin = [0.0] * n + + for dist, pt in nearest: + w = 1.0 / max(dist, 1.0) + total_weight += w + amps = pt.get('amp', []) + phases = pt.get('phase', []) + for j in range(min(n, len(amps), len(phases))): + weighted_amp[j] += w * amps[j] + ph_rad = phases[j] * tide_harmonics.DEG2RAD + weighted_phase_cos[j] += w * math.cos(ph_rad) + weighted_phase_sin[j] += w * math.sin(ph_rad) + + constituents = [] + for j, name in enumerate(const_names): + amp = weighted_amp[j] / total_weight + phase = math.atan2(weighted_phase_sin[j], weighted_phase_cos[j]) * tide_harmonics.RAD2DEG + if phase < 0: + phase += 360.0 + constituents.append({'name': name, 'amplitude': amp, 'phase': phase}) + + return constituents + + +def main(): + parser = argparse.ArgumentParser(description='Predict tides for a GPS location') + parser.add_argument('--lat', type=float, default=25.49) + parser.add_argument('--lon', type=float, default=-76.64) + parser.add_argument('--hours', type=int, default=36, + help='Hours to predict ahead') + args = parser.parse_args() + + grid_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + 'constituents', 'coastal_grid.json.gz') + if not os.path.exists(grid_path): + print(f"ERROR: Grid file not found: {grid_path}") + sys.exit(1) + + print(f"Loading coastal grid: {grid_path}") + grid = load_grid(grid_path) + print(f"Grid: {len(grid['points'])} points, " + f"constituents: {grid['constituents']}") + + print(f"\nInterpolating for ({args.lat}, {args.lon})...") + constituents = interpolate_constituents(grid, args.lat, args.lon) + + print(f"\nConstituents ({len(constituents)}):") + print(f" {'Name':<6} {'Amp (m)':>8} {'Phase':>8}") + print(f" {'-'*6} {'-'*8} {'-'*8}") + for c in constituents: + print(f" {c['name']:<6} {c['amplitude']:8.4f} {c['phase']:8.2f}") + + now = time.time() + start = now - 2 * 3600 + step = 360 + n_steps = (args.hours + 2) * 3600 // step + timestamps = [start + i * step for i in range(n_steps + 1)] + + heights = tide_harmonics.predict(constituents, timestamps) + extrema = tide_harmonics.find_extrema(timestamps, heights, min_amplitude=0.1) + + M_TO_FT = 3.28084 + + future_extrema = [(ts, h, t) for ts, h, t in extrema if ts > now] + + print(f"\n{'='*60}") + print(f"Tide predictions for ({args.lat}, {args.lon})") + print(f"Heights are MSL-relative (model Z0 = 0)") + print(f"{'='*60}") + print(f"\n {'Type':<6} {'Time (UTC)':<22} {'Time (Local)':<22} " + f"{'MSL (m)':>8} {'MSL (ft)':>9}") + print(f" {'-'*6} {'-'*22} {'-'*22} {'-'*8} {'-'*9}") + + for ts, h, etype in future_extrema[:8]: + dt_utc = datetime.fromtimestamp(ts, tz=timezone.utc) + dt_local = datetime.fromtimestamp(ts) + label = "High" if etype == "high" else "Low" + print(f" {label:<6} {dt_utc.strftime('%Y-%m-%d %H:%M %Z'):<22} " + f"{dt_local.strftime('%Y-%m-%d %H:%M'):<22} " + f"{h:8.3f} {h * M_TO_FT:9.2f}") + + print(f"\n--- Reference (published Harbour Island) ---") + print(f" Low 01:13 AM 0.69 ft (0.21 m MSL)") + print(f" High 07:36 AM 3.71 ft (1.13 m MSL)") + print(f" Low 01:46 PM 0.82 ft (0.25 m MSL)") + print(f" High 07:55 PM 3.51 ft (1.07 m MSL)") + print(f"\nNote: Published heights are above MLLW datum.") + print(f"Model heights are relative to MSL (Z0=0).") + print(f"To compare shape/timing, focus on time offsets") + print(f"and tidal range (high-low difference).") + + +if __name__ == '__main__': + main() diff --git a/dbus-tides/tools/requirements-dev.txt b/dbus-tides/tools/requirements-dev.txt new file mode 100644 index 0000000..f2f4ebb --- /dev/null +++ b/dbus-tides/tools/requirements-dev.txt @@ -0,0 +1,6 @@ +pyTMD +numpy +scipy +pyproj +h5netcdf +xarray diff --git a/dbus-tides/uninstall.sh b/dbus-tides/uninstall.sh new file mode 100644 index 0000000..ee64007 --- /dev/null +++ b/dbus-tides/uninstall.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Uninstall dbus-tides for Venus OS + +INSTALL_DIR="/data/dbus-tides" +SERVICE_LINK="dbus-tides" + +# Find service directory +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "Uninstalling dbus-tides..." + +# Stop and remove service +if [ -L "$SERVICE_DIR/$SERVICE_LINK" ] || [ -e "$SERVICE_DIR/$SERVICE_LINK" ]; then + echo "Stopping and removing service..." + svc -d "$SERVICE_DIR/$SERVICE_LINK" 2>/dev/null || true + rm -f "$SERVICE_DIR/$SERVICE_LINK" + rm -rf "$SERVICE_DIR/$SERVICE_LINK" +fi + +echo "Service removed. Config and data in $INSTALL_DIR are preserved." +echo "To remove everything: rm -rf $INSTALL_DIR /var/log/dbus-tides" diff --git a/dbus-vrm-history/.gitignore b/dbus-vrm-history/.gitignore new file mode 100644 index 0000000..5923ff5 --- /dev/null +++ b/dbus-vrm-history/.gitignore @@ -0,0 +1,24 @@ +# Build artifacts +*.tar.gz +*.sha256 + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg +*.egg-info/ +dist/ +build/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Venus OS runtime (created during installation) +ext/ diff --git a/dbus-vrm-history/README.md b/dbus-vrm-history/README.md new file mode 100644 index 0000000..a957aec --- /dev/null +++ b/dbus-vrm-history/README.md @@ -0,0 +1,48 @@ +# dbus-vrm-history + +Venus OS D-Bus service that proxies historical data from the [VRM](https://vrm.victronenergy.com/) cloud API and exposes it on D-Bus/MQTT for the frontend dashboard. + +## Features + +- Fetches historical stats from VRM cloud API (battery SOC, consumption, generator runtime, battery cycles) +- Auto-discovers VRM site ID from device portal ID +- Configurable refresh interval +- Exposes history data on D-Bus for use by venus-html5-app custom views + +## Architecture + +| Module | Purpose | +|--------|---------| +| `dbus-vrm-history.py` | Main service, D-Bus setup, request handling | +| `vrm_client.py` | VRM API client (stats, widgets, site discovery) | +| `config.py` | Service name, VRM URL, refresh interval, status codes | + +## Installation + +```bash +scp -r dbus-vrm-history root@:/data/ +ssh root@ +bash /data/dbus-vrm-history/install.sh +``` + +## Configuration + +Set your VRM API token after installation: + +```bash +dbus -y com.victronenergy.vrmhistory /Config/Token SetValue 'your-vrm-api-token' +``` + +## D-Bus Service + +- **Service name:** `com.victronenergy.vrmhistory` +- **Config paths:** `/Config/Token`, `/Config/SiteId`, `/Config/RefreshInterval` +- **History paths:** `/History/Soc`, `/History/Consumption`, `/History/GeneratorRuntime`, `/History/BatteryCycles` + +## Service Management + +```bash +svstat /service/dbus-vrm-history +svc -t /service/dbus-vrm-history # restart +tail -f /var/log/dbus-vrm-history/current | tai64nlocal +``` diff --git a/dbus-vrm-history/config.py b/dbus-vrm-history/config.py new file mode 100644 index 0000000..8f273ce --- /dev/null +++ b/dbus-vrm-history/config.py @@ -0,0 +1,27 @@ +""" +Configuration for dbus-vrm-history service +""" + +VRM_CONFIG = { + 'base_url': 'https://vrmapi.victronenergy.com/v2', + 'refresh_interval_minutes': 15, + 'request_timeout_seconds': 30, +} + +DBUS_CONFIG = { + 'service_name': 'com.victronenergy.vrmhistory', + 'settings_path': '/Settings/VrmHistory', +} + +LOGGING_CONFIG = { + 'level': 'INFO', + 'include_timestamp': False, +} + +STATUS = { + 'UNCONFIGURED': 0, + 'READY': 1, + 'FETCHING': 2, + 'ERROR': 3, + 'DISCOVERING': 4, +} diff --git a/dbus-vrm-history/dbus-vrm-history.py b/dbus-vrm-history/dbus-vrm-history.py new file mode 100644 index 0000000..a3ab0a6 --- /dev/null +++ b/dbus-vrm-history/dbus-vrm-history.py @@ -0,0 +1,419 @@ +#!/usr/bin/env python3 +""" +Venus OS VRM History Service + +Proxies VRM API historical data requests through D-Bus/MQTT. +Data is requested by the frontend via MQTT W/ topics, fetched from the +VRM cloud API, and published back as JSON strings on N/ topics. + +Service name: com.victronenergy.vrmhistory +MQTT paths: N/{portalId}/vrmhistory/0/... +""" + +import sys +import os +import json +import logging +import signal +import threading +from time import time, sleep + +sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'ext', 'velib_python')) +sys.path.insert(1, '/opt/victronenergy/velib_python') + +try: + from gi.repository import GLib +except ImportError: + print("ERROR: GLib not available. This script must run on Venus OS.") + sys.exit(1) + +try: + import dbus + from dbus.mainloop.glib import DBusGMainLoop + from vedbus import VeDbusService + from settingsdevice import SettingsDevice +except ImportError as e: + print(f"ERROR: Required module not available: {e}") + sys.exit(1) + +from config import DBUS_CONFIG, VRM_CONFIG, LOGGING_CONFIG, STATUS +from vrm_client import VrmClient + +VERSION = '1.0.0' +SERVICE_NAME = DBUS_CONFIG['service_name'] + + +class VrmHistoryService: + def __init__(self): + self._setup_logging() + self.logger = logging.getLogger('VrmHistory') + self.logger.info(f"Initializing VRM History Service v{VERSION}") + + self.bus = dbus.SystemBus() + self.vrm_client = None + self.portal_id = None + self._fetch_lock = threading.Lock() + + self._read_portal_id() + self._create_dbus_service() + self._setup_settings() + self._init_client() + + refresh_ms = VRM_CONFIG['refresh_interval_minutes'] * 60 * 1000 + GLib.timeout_add(refresh_ms, self._auto_refresh) + + self.logger.info("VRM History Service initialized") + + def _setup_logging(self): + level = getattr(logging, LOGGING_CONFIG['level'], logging.INFO) + fmt = '%(levelname)s %(name)s: %(message)s' + if LOGGING_CONFIG['include_timestamp']: + fmt = '%(asctime)s ' + fmt + logging.basicConfig(level=level, format=fmt, stream=sys.stdout) + + def _read_portal_id(self): + try: + with open('/data/venus/unique-id', 'r') as f: + self.portal_id = f.read().strip() + self.logger.info(f"Portal ID: {self.portal_id}") + except FileNotFoundError: + try: + bus = dbus.SystemBus() + obj = bus.get_object('com.victronenergy.platform', '/Vrmportalid') + self.portal_id = str(obj.GetValue()) + self.logger.info(f"Portal ID from dbus: {self.portal_id}") + except Exception as e: + self.logger.warning(f"Could not read portal ID: {e}") + self.portal_id = None + + def _create_dbus_service(self): + self.logger.info(f"Creating D-Bus service: {SERVICE_NAME}") + max_retries = 5 + retry_delay = 1.0 + + for attempt in range(max_retries): + try: + self.dbus_service = VeDbusService(SERVICE_NAME, self.bus) + break + except dbus.exceptions.NameExistsException: + if attempt < max_retries - 1: + self.logger.warning(f"D-Bus name exists, retrying ({attempt + 1}/{max_retries})") + sleep(retry_delay) + retry_delay *= 2 + else: + raise + + svc = self.dbus_service + svc.add_path('/Mgmt/ProcessName', 'dbus-vrm-history') + svc.add_path('/Mgmt/ProcessVersion', VERSION) + svc.add_path('/Mgmt/Connection', 'local') + svc.add_path('/DeviceInstance', 0) + svc.add_path('/ProductId', 0xFFFE) + svc.add_path('/ProductName', 'VRM History Proxy') + svc.add_path('/FirmwareVersion', VERSION) + svc.add_path('/Connected', 1) + + svc.add_path('/Status', STATUS['UNCONFIGURED']) + svc.add_path('/Error', '') + svc.add_path('/LastFetchTime', 0) + + svc.add_path('/Config/Token', '', writeable=True, + onchangecallback=self._on_token_changed) + svc.add_path('/Config/SiteId', 0, writeable=True, + onchangecallback=self._on_setting_changed) + svc.add_path('/Config/RefreshInterval', + VRM_CONFIG['refresh_interval_minutes'], + writeable=True, onchangecallback=self._on_setting_changed) + + for req_path in ['/Request/Soc', '/Request/Consumption', + '/Request/GeneratorRuntime', '/Request/BatteryCycles']: + svc.add_path(req_path, '', writeable=True, + onchangecallback=self._on_request) + + svc.add_path('/History/Soc', '') + svc.add_path('/History/Consumption', '') + svc.add_path('/History/GeneratorRuntime', '') + svc.add_path('/History/BatteryCycles', '') + + def _setup_settings(self): + try: + base = DBUS_CONFIG['settings_path'] + settings_def = { + 'Token': [base + '/Token', '', 0, 0], + 'SiteId': [base + '/SiteId', 0, 0, 999999999], + 'RefreshInterval': [base + '/RefreshInterval', + VRM_CONFIG['refresh_interval_minutes'], 1, 1440], + } + self.settings = SettingsDevice(self.bus, settings_def, + self._on_persistent_setting_changed) + + token = self.settings['Token'] + site_id = self.settings['SiteId'] + refresh = self.settings['RefreshInterval'] + + if token: + self.dbus_service['/Config/Token'] = token + if site_id: + self.dbus_service['/Config/SiteId'] = site_id + if refresh: + self.dbus_service['/Config/RefreshInterval'] = refresh + + self.logger.info("Persistent settings loaded") + except Exception as e: + self.logger.warning(f"Could not init persistent settings: {e}") + self.settings = None + + def _on_persistent_setting_changed(self, name, old, new): + self.logger.info(f"Setting changed: {name} = {new}") + + def _init_client(self): + token = self.dbus_service['/Config/Token'] + site_id = self.dbus_service['/Config/SiteId'] + + if not token: + self.dbus_service['/Status'] = STATUS['UNCONFIGURED'] + return + + self.vrm_client = VrmClient(token, site_id if site_id else None) + + if not site_id and self.portal_id: + self._discover_site_id() + elif site_id: + self.vrm_client.site_id = site_id + self.dbus_service['/Status'] = STATUS['READY'] + else: + self.dbus_service['/Status'] = STATUS['ERROR'] + self.dbus_service['/Error'] = 'No portal ID available for auto-discovery' + + def _discover_site_id(self): + def _do_discover(): + try: + GLib.idle_add(lambda: self._set_status(STATUS['DISCOVERING'])) + site_id = self.vrm_client.discover_site_id(self.portal_id) + self.vrm_client.site_id = site_id + + def _apply(sid): + self.dbus_service['/Config/SiteId'] = sid + if self.settings: + self.settings['SiteId'] = sid + self._set_status(STATUS['READY']) + self.logger.info(f"Auto-discovered site ID: {sid}") + return False + + GLib.idle_add(_apply, site_id) + except Exception as e: + self.logger.error(f"Site ID discovery failed: {e}") + GLib.idle_add(lambda: self._set_error(str(e))) + + t = threading.Thread(target=_do_discover, daemon=True) + t.start() + + def _set_status(self, status): + self.dbus_service['/Status'] = status + if status != STATUS['ERROR']: + self.dbus_service['/Error'] = '' + return False + + def _set_error(self, msg): + self.dbus_service['/Status'] = STATUS['ERROR'] + self.dbus_service['/Error'] = msg + return False + + def _on_token_changed(self, path, value): + self.logger.info("Token updated") + if self.settings: + self.settings['Token'] = value + self.vrm_client = VrmClient(value) + site_id = self.dbus_service['/Config/SiteId'] + if site_id: + self.vrm_client.site_id = site_id + self.dbus_service['/Status'] = STATUS['READY'] + elif self.portal_id: + self._discover_site_id() + return True + + def _on_setting_changed(self, path, value): + if path == '/Config/SiteId' and self.vrm_client: + self.vrm_client.site_id = value + if self.settings: + self.settings['SiteId'] = value + if value: + self.dbus_service['/Status'] = STATUS['READY'] + elif path == '/Config/RefreshInterval': + if self.settings: + self.settings['RefreshInterval'] = value + return True + + def _on_request(self, path, value): + if not value: + return True + + request_map = { + '/Request/Soc': self._fetch_soc, + '/Request/Consumption': self._fetch_consumption, + '/Request/GeneratorRuntime': self._fetch_generator_runtime, + '/Request/BatteryCycles': self._fetch_battery_cycles, + } + + handler = request_map.get(path) + if not handler: + return True + + try: + params = json.loads(value) if isinstance(value, str) else value + except (json.JSONDecodeError, TypeError): + self.logger.error(f"Invalid request JSON on {path}: {value}") + return True + + def _bg(): + with self._fetch_lock: + try: + GLib.idle_add(lambda: self._set_status(STATUS['FETCHING'])) + handler(params) + GLib.idle_add(self._fetch_complete) + except Exception as e: + self.logger.error(f"Fetch error for {path}: {e}") + GLib.idle_add(lambda: self._set_error(str(e))) + + t = threading.Thread(target=_bg, daemon=True) + t.start() + return True + + def _fetch_complete(self): + self.dbus_service['/Status'] = STATUS['READY'] + self.dbus_service['/LastFetchTime'] = int(time()) + return False + + def _fetch_soc(self, params): + start = params.get('start', int(time()) - 86400) + end = params.get('end', int(time())) + resp = self.vrm_client.get_stats('live_feed', ['bs'], start, end) + + records = resp.get('records', {}) + bs_data = records.get('bs', []) + result = [{"ts": int(pt[0]), "soc": round(pt[1], 1)} + for pt in bs_data if len(pt) >= 2] + + data = json.dumps(result) + GLib.idle_add(lambda: self._publish('/History/Soc', data)) + + def _fetch_consumption(self, params): + start = params.get('start', int(time()) - 604800) + end = params.get('end', int(time())) + interval = params.get('interval', 'days') + resp = self.vrm_client.get_stats('kwh', ['kwh'], start, end, interval) + + records = resp.get('records', {}) + kwh_data = records.get('kwh', []) + + result = [] + for pt in kwh_data: + if len(pt) < 2: + continue + entry = { + "ts": int(pt[0]), + "genKwh": round(pt[1] if len(pt) > 1 else 0, 2), + "gridKwh": round(pt[2] if len(pt) > 2 else 0, 2), + "totalKwh": round((pt[1] if len(pt) > 1 else 0) + + (pt[2] if len(pt) > 2 else 0), 2), + } + result.append(entry) + + data = json.dumps(result) + GLib.idle_add(lambda: self._publish('/History/Consumption', data)) + + def _fetch_generator_runtime(self, params): + start = params.get('start', int(time()) - 604800) + end = params.get('end', int(time())) + + try: + resp = self.vrm_client.get_widget('HistoricData') + hist = resp.get('records', {}).get('data', []) + result = [] + for day in hist: + ts = day.get('timestamp', 0) + if ts < start or ts > end: + continue + hours = day.get('kwh', {}).get('generator_runtime_hours', 0) + starts = day.get('kwh', {}).get('generator_starts', 0) + result.append({"ts": ts, "hours": round(hours, 1), + "starts": int(starts)}) + except Exception: + result = [] + + data = json.dumps(result) + GLib.idle_add(lambda: self._publish('/History/GeneratorRuntime', data)) + + def _fetch_battery_cycles(self, params): + start = params.get('start', int(time()) - 604800) + end = params.get('end', int(time())) + + try: + resp = self.vrm_client.get_widget('HistoricData') + hist = resp.get('records', {}).get('data', []) + result = [] + for day in hist: + ts = day.get('timestamp', 0) + if ts < start or ts > end: + continue + charge = day.get('kwh', {}).get('battery_charge', 0) + discharge = day.get('kwh', {}).get('battery_discharge', 0) + result.append({"ts": ts, "chargeKwh": round(charge, 2), + "dischargeKwh": round(discharge, 2)}) + except Exception: + result = [] + + data = json.dumps(result) + GLib.idle_add(lambda: self._publish('/History/BatteryCycles', data)) + + def _publish(self, path, value): + self.dbus_service[path] = value + return False + + def _auto_refresh(self): + if self.dbus_service['/Status'] == STATUS['UNCONFIGURED']: + return True + if not self.vrm_client or not self.vrm_client.site_id: + return True + + self.logger.info("Auto-refresh triggered") + now = int(time()) + + def _bg(): + with self._fetch_lock: + try: + GLib.idle_add(lambda: self._set_status(STATUS['FETCHING'])) + self._fetch_soc({'start': now - 86400, 'end': now}) + self._fetch_consumption({'start': now - 604800, 'end': now, + 'interval': 'days'}) + self._fetch_generator_runtime({'start': now - 604800, 'end': now}) + self._fetch_battery_cycles({'start': now - 604800, 'end': now}) + GLib.idle_add(self._fetch_complete) + except Exception as e: + self.logger.error(f"Auto-refresh error: {e}") + GLib.idle_add(lambda: self._set_error(str(e))) + + t = threading.Thread(target=_bg, daemon=True) + t.start() + return True + + +def main(): + DBusGMainLoop(set_as_default=True) + + service = VrmHistoryService() + + def signal_handler(signum, frame): + logging.getLogger('VrmHistory').info("Received signal, exiting...") + mainloop.quit() + + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + + mainloop = GLib.MainLoop() + logging.getLogger('VrmHistory').info("Starting main loop") + mainloop.run() + + +if __name__ == '__main__': + main() diff --git a/dbus-vrm-history/install.sh b/dbus-vrm-history/install.sh new file mode 100644 index 0000000..bffd6b1 --- /dev/null +++ b/dbus-vrm-history/install.sh @@ -0,0 +1,113 @@ +#!/bin/bash +set -e + +INSTALL_DIR="/data/dbus-vrm-history" +VELIB_DIR="" +if [ -d "/opt/victronenergy/velib_python" ]; then + VELIB_DIR="/opt/victronenergy/velib_python" +else + for candidate in \ + "/opt/victronenergy/dbus-systemcalc-py/ext/velib_python" \ + "/opt/victronenergy/dbus-generator/ext/velib_python" \ + "/opt/victronenergy/dbus-mqtt/ext/velib_python" \ + "/opt/victronenergy/dbus-digitalinputs/ext/velib_python" \ + "/opt/victronenergy/vrmlogger/ext/velib_python" + do + if [ -d "$candidate" ] && [ -f "$candidate/vedbus.py" ]; then + VELIB_DIR="$candidate" + break + fi + done +fi + +if [ -z "$VELIB_DIR" ]; then + VEDBUS_PATH=$(find /opt/victronenergy -name "vedbus.py" -path "*/velib_python/*" 2>/dev/null | head -1) + if [ -n "$VEDBUS_PATH" ]; then + VELIB_DIR=$(dirname "$VEDBUS_PATH") + fi +fi + +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +elif [ -L "/service" ]; then + SERVICE_DIR=$(readlink -f /service) +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "==================================================" +echo "VRM History Service - Installation" +echo "==================================================" + +if [ ! -d "$SERVICE_DIR" ]; then + echo "ERROR: Not a Venus OS device."; exit 1 +fi + +echo "Service directory: $SERVICE_DIR" + +if [ ! -f "$INSTALL_DIR/dbus-vrm-history.py" ]; then + echo "ERROR: Files not found in $INSTALL_DIR"; exit 1 +fi + +echo "1. Making scripts executable..." +chmod +x "$INSTALL_DIR/service/run" +chmod +x "$INSTALL_DIR/service/log/run" +chmod +x "$INSTALL_DIR/dbus-vrm-history.py" + +echo "2. Creating velib_python symlink..." +if [ -z "$VELIB_DIR" ]; then + echo "ERROR: Could not find velib_python"; exit 1 +fi +echo " Found: $VELIB_DIR" +mkdir -p "$INSTALL_DIR/ext" +if [ -L "$INSTALL_DIR/ext/velib_python" ]; then + CURRENT_TARGET=$(readlink "$INSTALL_DIR/ext/velib_python") + if [ "$CURRENT_TARGET" != "$VELIB_DIR" ]; then + rm "$INSTALL_DIR/ext/velib_python" + fi +fi +if [ ! -L "$INSTALL_DIR/ext/velib_python" ]; then + ln -s "$VELIB_DIR" "$INSTALL_DIR/ext/velib_python" +fi + +echo "3. Creating service symlink..." +if [ -L "$SERVICE_DIR/dbus-vrm-history" ]; then + rm "$SERVICE_DIR/dbus-vrm-history" +fi +if [ -e "$SERVICE_DIR/dbus-vrm-history" ]; then + rm -rf "$SERVICE_DIR/dbus-vrm-history" +fi +ln -s "$INSTALL_DIR/service" "$SERVICE_DIR/dbus-vrm-history" + +echo "4. Creating log directory..." +mkdir -p /var/log/dbus-vrm-history + +echo "5. Setting up rc.local..." +RC_LOCAL="/data/rc.local" +if [ ! -f "$RC_LOCAL" ]; then + echo "#!/bin/bash" > "$RC_LOCAL" + chmod +x "$RC_LOCAL" +fi +if ! grep -q "dbus-vrm-history" "$RC_LOCAL"; then + echo "" >> "$RC_LOCAL" + echo "# VRM History Service" >> "$RC_LOCAL" + echo "if [ ! -L $SERVICE_DIR/dbus-vrm-history ]; then" >> "$RC_LOCAL" + echo " ln -s /data/dbus-vrm-history/service $SERVICE_DIR/dbus-vrm-history" >> "$RC_LOCAL" + echo "fi" >> "$RC_LOCAL" +fi + +echo "6. Activating..." +sleep 2 + +echo "" +echo "==================================================" +echo "Installation complete!" +echo "==================================================" +echo "" +echo "Configure your VRM token:" +echo " dbus -y com.victronenergy.vrmhistory /Config/Token SetValue 'your-token'" +echo "" +echo "MQTT: N//vrmhistory/0/..." +echo "" diff --git a/dbus-vrm-history/service/log/run b/dbus-vrm-history/service/log/run new file mode 100644 index 0000000..7f1ae3b --- /dev/null +++ b/dbus-vrm-history/service/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec multilog t s99999 n8 /var/log/dbus-vrm-history diff --git a/dbus-vrm-history/service/run b/dbus-vrm-history/service/run new file mode 100644 index 0000000..7e62cf2 --- /dev/null +++ b/dbus-vrm-history/service/run @@ -0,0 +1,5 @@ +#!/bin/sh +exec 2>&1 +cd /data/dbus-vrm-history +export PYTHONPATH="/data/dbus-vrm-history/ext/velib_python:$PYTHONPATH" +exec python3 /data/dbus-vrm-history/dbus-vrm-history.py diff --git a/dbus-vrm-history/vrm_client.py b/dbus-vrm-history/vrm_client.py new file mode 100644 index 0000000..36e382f --- /dev/null +++ b/dbus-vrm-history/vrm_client.py @@ -0,0 +1,92 @@ +""" +VRM API client using urllib (available on Venus OS without pip). +""" + +import json +import logging +from urllib.request import Request, urlopen +from urllib.error import HTTPError, URLError + +logger = logging.getLogger('VrmClient') + + +class VrmClient: + BASE_URL = "https://vrmapi.victronenergy.com/v2" + + def __init__(self, token, site_id=None): + self.token = token + self.site_id = site_id + + def _request(self, path, params=None): + url = f"{self.BASE_URL}{path}" + if params: + query = "&".join(f"{k}={v}" for k, v in params.items()) + url = f"{url}?{query}" + + req = Request(url) + req.add_header("x-authorization", f"Token {self.token}") + req.add_header("Content-Type", "application/json") + + try: + with urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode("utf-8")) + except HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + logger.error(f"VRM API HTTP {e.code}: {body[:200]}") + raise + except URLError as e: + logger.error(f"VRM API connection error: {e.reason}") + raise + + def discover_site_id(self, portal_id): + """Auto-discover numeric siteId from the GX device portal ID. + + 1. GET /users/me -> idUser + 2. GET /users/{idUser}/installations -> find matching identifier + """ + me = self._request("/users/me") + id_user = me.get("idUser") or me.get("user", {}).get("idUser") + if not id_user: + raise ValueError("Could not determine idUser from VRM API") + + installs = self._request(f"/users/{id_user}/installations") + records = installs.get("records", []) + + for inst in records: + if inst.get("identifier") == portal_id: + logger.info(f"Found VRM site: {inst.get('name')} (id={inst.get('idSite')})") + return inst["idSite"] + + available = [f"{r.get('name')}({r.get('identifier')})" for r in records[:5]] + raise ValueError( + f"No installation matches portal ID '{portal_id}'. " + f"Available: {', '.join(available)}" + ) + + def get_stats(self, stat_type, attribute_codes, start, end, interval=None): + """GET /installations/{siteId}/stats + + stat_type: 'live_feed' or 'kwh' + attribute_codes: list of VRM attribute codes (e.g. ['bs'] for battery SOC) + """ + if not self.site_id: + raise ValueError("site_id not configured") + + params = {"type": stat_type, "start": str(start), "end": str(end)} + for i, code in enumerate(attribute_codes): + params[f"attributeCodes[{i}]"] = code + if interval: + params["interval"] = interval + + return self._request(f"/installations/{self.site_id}/stats", params) + + def get_widget(self, widget_name, instance=None): + """GET /installations/{siteId}/widgets/{widgetName}""" + if not self.site_id: + raise ValueError("site_id not configured") + + params = {} + if instance is not None: + params["instance"] = str(instance) + + return self._request(f"/installations/{self.site_id}/widgets/{widget_name}", params) diff --git a/dbus-windy-station/.gitignore b/dbus-windy-station/.gitignore new file mode 100644 index 0000000..e2d30b8 --- /dev/null +++ b/dbus-windy-station/.gitignore @@ -0,0 +1,27 @@ +# Build artifacts +*.tar.gz +*.sha256 + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg +*.egg-info/ +dist/ +build/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Venus OS runtime (created during installation) +ext/ + +# Sensitive config (use station_config.example.json as template) +station_config.json diff --git a/dbus-windy-station/README.md b/dbus-windy-station/README.md new file mode 100644 index 0000000..90c9635 --- /dev/null +++ b/dbus-windy-station/README.md @@ -0,0 +1,183 @@ +# Windy Station for Venus OS + +Uploads weather data from Venus OS to [Windy.com](https://windy.com) weather station network using the v2 API. + +Reads weather data from the D-Bus meteo service (published by the Raymarine integration) and GPS position from the system GPS service. Provides a settings GUI on the Cerbo GX display via either the v1 GUI (legacy) or v2 GUI plugin system (Venus OS v3.70+). + +## Features + +- Uploads wind speed (5-minute running average), wind gusts (peak 3-sample average over 10 minutes), wind direction, temperature, and barometric pressure +- Updates station GPS location automatically (on startup, every 60 minutes, or when GPS moves > 200 ft) +- All settings configurable via the Venus OS touchscreen GUI or MQTT +- Settings persist across service restarts via Venus localsettings +- Survives firmware updates via rc.local persistence + +## Prerequisites + +- Venus OS device (Cerbo GX, Venus GX, Raspberry Pi with Venus OS) +- Raymarine weather data publishing to D-Bus (via raymarine-venus package) +- Windy.com station credentials (see Setup below) + +## Windy.com Setup + +1. Go to [stations.windy.com](https://stations.windy.com/) +2. Create an account or sign in +3. Go to **My Stations** and add a new station +4. Note your **Station ID** and **Station Password** from the station detail page +5. Go to **API Keys** and create an API key (for station location updates) + +## Installation + +### Quick Install + +```bash +scp dbus-windy-station-*.tar.gz root@:/data/ +ssh root@ +cd /data && tar -xzf dbus-windy-station-*.tar.gz +bash /data/dbus-windy-station/install.sh +``` + +Then install the GUI for your Venus OS version: + +**GUI v2 plugin (Venus OS v3.70+, recommended):** + +```bash +bash /data/dbus-windy-station/install_gui_v2.sh +``` + +The plugin appears under **Settings > Integrations > UI Plugins > Windy Station**. No system files are patched; the plugin survives firmware updates. + +**GUI v1 legacy (Venus OS < v3.70):** + +```bash +bash /data/dbus-windy-station/install_gui.sh +``` + +Adds an entry under **Settings > Windy Station**. Must be re-run after firmware updates. + +### Configuration + +#### Option 1: Config File (recommended for initial setup) + +Create `/data/dbus-windy-station/station_config.json`: + +```json +{ + "api_key": "your-api-key", + "station_id": "pws-station-001", + "station_password": "your-station-password", + "update_interval": 300, + "name": "My Weather Station", + "share_option": "public", + "station_type": "Boat", + "elev_m": 0, + "agl_wind": 5, + "agl_temp": 5 +} +``` + +The config file is read on every service startup and **overrides** any GUI/localsettings values. This makes it easy to deploy a known configuration. Both snake_case and camelCase field names are accepted. + +#### Option 2: GUI Configuration + +Navigate to **Settings > Windy Station** (v1) or **Settings > Integrations > UI Plugins > Windy Station** (v2) on the Cerbo display: + +1. Enter your **API Key**, **Station ID**, and **Station Password** +2. Set **Station Name**, **Station Type**, and **Share Option** +3. Set sensor heights (**Wind sensor height**, **Temp sensor height**) +4. Enable uploads with the toggle switch +5. Optionally adjust the upload interval (default: 300s / 5 minutes) + +#### Option 3: MQTT + +``` +W//windystation/0/Settings/ApiKey "your-api-key" +W//windystation/0/Settings/StationId "pws-station-001" +W//windystation/0/Settings/StationPassword "your-password" +W//windystation/0/Settings/Name "My Station" +W//windystation/0/Settings/ShareOption "public" +W//windystation/0/Settings/StationType "Boat" +W//windystation/0/Settings/ElevM 0 +W//windystation/0/Settings/AglWind 5 +W//windystation/0/Settings/AglTemp 5 +W//windystation/0/Settings/Enabled 1 +``` + +## Building + +```bash +./build-package.sh --version 1.0.0 +``` + +Creates `dbus-windy-station-1.0.0.tar.gz` ready for deployment. + +## Service Management + +```bash +# Check status +svstat /service/dbus-windy-station + +# View logs +tail -F /var/log/dbus-windy-station/current | tai64nlocal + +# Restart +svc -t /service/dbus-windy-station + +# Stop +svc -d /service/dbus-windy-station + +# Start +svc -u /service/dbus-windy-station +``` + +## Uninstall + +```bash +bash /data/dbus-windy-station/uninstall.sh +# To remove all files: +rm -rf /data/dbus-windy-station /var/log/dbus-windy-station +``` + +## D-Bus / MQTT Paths + +| Path | Type | Description | +|------|------|-------------| +| `/Status` | read | 0=Idle, 1=Active, 2=Uploading, 3=Error | +| `/WindSpeed` | read | 5-minute average wind speed (m/s) | +| `/WindGust` | read | Peak gust (m/s) | +| `/WindDirection` | read | Wind direction (degrees) | +| `/Temperature` | read | Air temperature (°C) | +| `/Pressure` | read | Barometric pressure (hPa) | +| `/LastReportTimeAgo` | read | Time since last upload (e.g., "3m") | +| `/Settings/Enabled` | r/w | Enable/disable uploads (0/1) | +| `/Settings/ApiKey` | r/w | Windy API key (for station management) | +| `/Settings/StationId` | r/w | Windy station ID | +| `/Settings/StationPassword` | r/w | Windy station password (for observations) | +| `/Settings/UploadInterval` | r/w | Upload interval in seconds (300-900) | +| `/Settings/Name` | r/w | Station display name on Windy.com | +| `/Settings/ShareOption` | r/w | public, only_windy, or private | +| `/Settings/StationType` | r/w | Station type (e.g., "Boat", "Airmar WX200") | +| `/Settings/ElevM` | r/w | Station elevation in meters | +| `/Settings/AglWind` | r/w | Wind sensor height above ground (meters) | +| `/Settings/AglTemp` | r/w | Temp sensor height above ground (meters) | + +## API Details + +Uses the Windy Stations API v2 (effective January 2026): + +- **Station update**: `PUT /api/v2/pws/{station_id}` with `windy-api-key` header +- **Observation upload**: `GET /api/v2/observation/update?id=...&PASSWORD=...&wind=...` + +Pressure is sent in Pascals (hPa * 100), wind in m/s, temperature in Celsius. + +## After Firmware Updates + +The dbus-windy-station service is preserved via rc.local. + +**GUI v2 plugin (v3.70+):** No action needed -- the plugin is stored under `/data/apps/` and survives firmware updates automatically. + +**GUI v1 legacy:** GUI modifications are lost on firmware updates. Re-run: + +```bash +bash /data/dbus-windy-station/install_gui.sh +``` diff --git a/dbus-windy-station/build-package.sh b/dbus-windy-station/build-package.sh new file mode 100755 index 0000000..7c64fcb --- /dev/null +++ b/dbus-windy-station/build-package.sh @@ -0,0 +1,184 @@ +#!/bin/bash +# +# Build script for Windy Station Venus OS package +# +# Creates a tar.gz package that can be: +# 1. Copied to a Venus OS device (Cerbo GX, Venus GX, etc.) +# 2. Untarred to /data/ +# 3. Installed by running install.sh +# +# Usage: +# ./build-package.sh # Creates package with default name +# ./build-package.sh --version 1.0.0 # Creates package with version in name +# ./build-package.sh --output /path/ # Specify output directory +# +# Installation on Venus OS: +# scp dbus-windy-station-*.tar.gz root@:/data/ +# ssh root@ +# cd /data && tar -xzf dbus-windy-station-*.tar.gz +# bash /data/dbus-windy-station/install.sh +# bash /data/dbus-windy-station/install_gui.sh # v1 GUI (legacy) +# bash /data/dbus-windy-station/install_gui_v2.sh # v2 GUI plugin (v3.70+) +# Configure via Settings > Windy Station (v1) or Settings > Integrations (v2) +# + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +VERSION="1.0.0" +OUTPUT_DIR="$SCRIPT_DIR" +PACKAGE_NAME="dbus-windy-station" + +while [[ $# -gt 0 ]]; do + case $1 in + --version|-v) + VERSION="$2" + shift 2 + ;; + --output|-o) + OUTPUT_DIR="$2" + shift 2 + ;; + --help|-h) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " -v, --version VERSION Set package version (default: 1.0.0)" + echo " -o, --output PATH Output directory (default: script directory)" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +BUILD_DATE=$(date -u +"%Y-%m-%d %H:%M:%S UTC") +BUILD_TIMESTAMP=$(date +%Y%m%d%H%M%S) + +BUILD_DIR=$(mktemp -d) +PACKAGE_DIR="$BUILD_DIR/$PACKAGE_NAME" + +echo "==================================================" +echo "Building $PACKAGE_NAME package" +echo "==================================================" +echo "Version: $VERSION" +echo "Build date: $BUILD_DATE" +echo "Source: $SCRIPT_DIR" +echo "Output: $OUTPUT_DIR" +echo "" + +echo "1. Creating package structure..." +mkdir -p "$PACKAGE_DIR" +mkdir -p "$PACKAGE_DIR/service/log" +mkdir -p "$PACKAGE_DIR/qml" +mkdir -p "$PACKAGE_DIR/gui-v2-source" + +[ "$(uname)" = "Darwin" ] && export COPYFILE_DISABLE=1 + +echo "2. Copying application files..." +cp "$SCRIPT_DIR/windy_station.py" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/config.py" "$PACKAGE_DIR/" + +echo "3. Copying service files..." +cp "$SCRIPT_DIR/service/run" "$PACKAGE_DIR/service/" +cp "$SCRIPT_DIR/service/log/run" "$PACKAGE_DIR/service/log/" + +echo "4. Copying GUI files..." +if [ -f "$SCRIPT_DIR/qml/PageSettingsWindyStation.qml" ]; then + cp "$SCRIPT_DIR/qml/PageSettingsWindyStation.qml" "$PACKAGE_DIR/qml/" +fi +if [ -f "$SCRIPT_DIR/gui-v2-source/WindyStation_PageSettings.qml" ]; then + cp "$SCRIPT_DIR/gui-v2-source/WindyStation_PageSettings.qml" "$PACKAGE_DIR/gui-v2-source/" +fi + +echo "5. Copying installation scripts..." +cp "$SCRIPT_DIR/install.sh" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/uninstall.sh" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/install_gui.sh" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/install_gui_v2.sh" "$PACKAGE_DIR/" +cp "$SCRIPT_DIR/compile_gui_v2_plugin.py" "$PACKAGE_DIR/" + +echo "6. Copying documentation..." +if [ -f "$SCRIPT_DIR/README.md" ]; then + cp "$SCRIPT_DIR/README.md" "$PACKAGE_DIR/" +fi + +echo "7. Creating version info..." +cat > "$PACKAGE_DIR/VERSION" << EOF +Package: $PACKAGE_NAME +Version: $VERSION +Build Date: $BUILD_DATE +Build Timestamp: $BUILD_TIMESTAMP + +Installation: + 1. Copy to Venus OS: scp $PACKAGE_NAME-$VERSION.tar.gz root@:/data/ + 2. SSH to device: ssh root@ + 3. Extract: cd /data && tar -xzf $PACKAGE_NAME-$VERSION.tar.gz + 4. Install: bash /data/dbus-windy-station/install.sh + 5. GUI (v1): bash /data/dbus-windy-station/install_gui.sh + GUI (v2): bash /data/dbus-windy-station/install_gui_v2.sh + 6. Configure via Settings > Windy Station (v1) or Settings > Integrations (v2) +EOF + +echo "8. Setting permissions..." +chmod +x "$PACKAGE_DIR/windy_station.py" +chmod +x "$PACKAGE_DIR/install.sh" +chmod +x "$PACKAGE_DIR/uninstall.sh" +chmod +x "$PACKAGE_DIR/install_gui.sh" +chmod +x "$PACKAGE_DIR/install_gui_v2.sh" +chmod +x "$PACKAGE_DIR/service/run" +chmod +x "$PACKAGE_DIR/service/log/run" + +mkdir -p "$OUTPUT_DIR" + +TARBALL_NAME="$PACKAGE_NAME-$VERSION.tar.gz" +OUTPUT_DIR_ABS="$(cd "$OUTPUT_DIR" && pwd)" +TARBALL_PATH="$OUTPUT_DIR_ABS/$TARBALL_NAME" + +echo "9. Creating package archive..." +cd "$BUILD_DIR" +if [ "$(uname)" = "Darwin" ]; then + if command -v xattr >/dev/null 2>&1; then + xattr -cr "$PACKAGE_NAME" + fi +fi +tar --format=ustar -czf "$TARBALL_PATH" "$PACKAGE_NAME" + +if command -v sha256sum >/dev/null 2>&1; then + CHECKSUM=$(sha256sum "$TARBALL_PATH" | cut -d' ' -f1) +else + CHECKSUM=$(shasum -a 256 "$TARBALL_PATH" | cut -d' ' -f1) +fi +echo "$CHECKSUM $TARBALL_NAME" > "$OUTPUT_DIR_ABS/$TARBALL_NAME.sha256" + +echo "10. Cleaning up..." +rm -rf "$BUILD_DIR" + +if [ "$(uname)" = "Darwin" ]; then + FILE_SIZE=$(du -h "$TARBALL_PATH" | cut -f1) +else + FILE_SIZE=$(du -h "$TARBALL_PATH" | cut -f1) +fi + +echo "" +echo "==================================================" +echo "Build complete!" +echo "==================================================" +echo "" +echo "Package: $TARBALL_PATH" +echo "Size: $FILE_SIZE" +echo "SHA256: $CHECKSUM" +echo "" +echo "Installation on Venus OS:" +echo " scp $TARBALL_PATH root@:/data/" +echo " ssh root@" +echo " cd /data" +echo " tar -xzf $TARBALL_NAME" +echo " bash /data/dbus-windy-station/install.sh" +echo " bash /data/dbus-windy-station/install_gui.sh # v1 GUI (legacy)" +echo " bash /data/dbus-windy-station/install_gui_v2.sh # v2 GUI plugin (v3.70+)" +echo "" diff --git a/dbus-windy-station/compile_gui_v2_plugin.py b/dbus-windy-station/compile_gui_v2_plugin.py new file mode 100644 index 0000000..d84c2c4 --- /dev/null +++ b/dbus-windy-station/compile_gui_v2_plugin.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +""" +Standalone GUI v2 plugin compiler for Venus OS. + +Generates the plugin JSON file without requiring Qt SDK tools +(lupdate, lrelease, rcc). Builds a Qt binary resource (.rcc) +natively in Python and embeds it in the JSON descriptor. + +Compatible with the gui-v2 plugin loader on Venus OS v3.70+. +""" + +import argparse +import base64 +import io +import json +import os +import struct +import sys + + +def qt_hash(name): + """Compute Qt resource name hash (matches qt_hash in QResourceRoot).""" + h = 0 + for ch in name: + h = ((h << 4) + ord(ch)) & 0xFFFFFFFF + h ^= (h & 0xF0000000) >> 23 + h &= 0x0FFFFFFF + return h + + +def encode_name(name): + """Encode a name entry for the RCC names section.""" + buf = io.BytesIO() + buf.write(struct.pack('>H', len(name))) + buf.write(struct.pack('>I', qt_hash(name))) + buf.write(name.encode('utf-16-be')) + return buf.getvalue() + + +def build_rcc(prefix, files): + """ + Build a Qt binary resource (.rcc) containing the given files + under the specified prefix. + + Tree nodes are sorted by name hash for binary search compatibility + with QResourceRoot::findNode(). + """ + sorted_files = sorted(files.keys(), key=lambda n: qt_hash(n)) + + names_buf = io.BytesIO() + name_offsets = {} + + name_offsets[""] = names_buf.tell() + names_buf.write(encode_name("")) + + name_offsets[prefix] = names_buf.tell() + names_buf.write(encode_name(prefix)) + + for fname in sorted_files: + name_offsets[fname] = names_buf.tell() + names_buf.write(encode_name(fname)) + + names_bytes = names_buf.getvalue() + + data_buf = io.BytesIO() + data_offsets = {} + + for fname in sorted_files: + data_offsets[fname] = data_buf.tell() + content = files[fname] + data_buf.write(struct.pack('>I', len(content))) + data_buf.write(content) + + data_bytes = data_buf.getvalue() + + tree_buf = io.BytesIO() + + tree_buf.write(struct.pack('>I', name_offsets[""])) + tree_buf.write(struct.pack('>H', 0x02)) + tree_buf.write(struct.pack('>I', 1)) + tree_buf.write(struct.pack('>I', 1)) + + tree_buf.write(struct.pack('>I', name_offsets[prefix])) + tree_buf.write(struct.pack('>H', 0x02)) + tree_buf.write(struct.pack('>I', len(sorted_files))) + tree_buf.write(struct.pack('>I', 2)) + + for fname in sorted_files: + tree_buf.write(struct.pack('>I', name_offsets[fname])) + tree_buf.write(struct.pack('>H', 0x00)) + tree_buf.write(struct.pack('>H', 0)) + tree_buf.write(struct.pack('>H', 0)) + tree_buf.write(struct.pack('>I', data_offsets[fname])) + + tree_bytes = tree_buf.getvalue() + + HEADER_SIZE = 20 + tree_offset = HEADER_SIZE + data_offset = tree_offset + len(tree_bytes) + names_offset = data_offset + len(data_bytes) + + out = io.BytesIO() + out.write(b'qres') + out.write(struct.pack('>I', 1)) + out.write(struct.pack('>I', tree_offset)) + out.write(struct.pack('>I', data_offset)) + out.write(struct.pack('>I', names_offset)) + out.write(tree_bytes) + out.write(data_bytes) + out.write(names_bytes) + + return out.getvalue() + + +def main(): + parser = argparse.ArgumentParser( + prog='compile-gui-v2-plugin', + description='Compile a GUI v2 plugin JSON without Qt SDK tools') + parser.add_argument('-n', '--name', required=True) + parser.add_argument('-v', '--version', default='1.0') + parser.add_argument('-z', '--min-required-version', default='') + parser.add_argument('-x', '--max-required-version', default='') + parser.add_argument('-s', '--settings', default='') + + args = parser.parse_args() + + files = {} + for fname in os.listdir('.'): + if fname.endswith(('.qml', '.svg', '.png')): + with open(fname, 'rb') as f: + files[fname] = f.read() + + if not files: + print("ERROR: No .qml/.svg/.png files found in current directory", + file=sys.stderr) + sys.exit(1) + + print("--- files to bundle:") + for fname in sorted(files.keys()): + print(" %s (%d bytes)" % (fname, len(files[fname]))) + + if args.settings and args.settings not in files: + print("ERROR: Settings page '%s' not found" % args.settings, + file=sys.stderr) + sys.exit(1) + + print("--- building resource binary") + rcc_data = build_rcc(args.name, files) + + print("--- base64 encoding resource (%d bytes)" % len(rcc_data)) + resource = base64.b64encode(rcc_data).decode('utf-8') + + integrations = [] + if args.settings: + integrations.append({ + "type": 1, + "url": "qrc:/%s/%s" % (args.name, args.settings) + }) + + output = { + "name": args.name, + "version": args.version, + "minRequiredVersion": args.min_required_version, + "maxRequiredVersion": args.max_required_version, + "translations": [], + "integrations": integrations, + "resource": resource + } + + output_file = "%s.json" % args.name + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(output, f, indent=4) + f.write('\n') + + print("--- wrote %s" % output_file) + print("--- done!") + + +if __name__ == '__main__': + main() diff --git a/dbus-windy-station/config.py b/dbus-windy-station/config.py new file mode 100644 index 0000000..54f3d83 --- /dev/null +++ b/dbus-windy-station/config.py @@ -0,0 +1,62 @@ +""" +Configuration for Windy Station Venus OS service. + +All tunable parameters in one place for easy adjustment. +""" + +# ============================================================================= +# D-BUS SERVICE CONFIGURATION +# ============================================================================= + +SERVICE_NAME = 'com.victronenergy.windystation' + +# ============================================================================= +# WINDY API CONFIGURATION (v2 - effective January 2026) +# ============================================================================= + +API_BASE = 'https://stations.windy.com/api/v2' + +# ============================================================================= +# TIMING CONFIGURATION +# ============================================================================= + +# Minimum interval between observation uploads (seconds) +# Windy recommends 5-minute intervals +OBSERVATION_MIN_INTERVAL = 300 + +# Interval between station metadata/location updates (seconds) +STATION_UPDATE_INTERVAL = 3600 # 60 minutes + +# GPS movement threshold before triggering a station update +GPS_MOVEMENT_THRESHOLD_FEET = 200.0 +GPS_MOVEMENT_THRESHOLD_METERS = GPS_MOVEMENT_THRESHOLD_FEET * 0.3048 + +# ============================================================================= +# WIND CALCULATION CONFIGURATION +# ============================================================================= + +# Window for computing running average wind speed (seconds) +WIND_AVG_WINDOW = 300 # 5 minutes + +# Window for gust tracking (seconds) +GUST_WINDOW = 600 # 10 minutes + +# Window for wind direction history buffer (seconds) +WIND_DIR_HISTORY_WINDOW = 1800 # 30 minutes + +# ============================================================================= +# PATH CONFIGURATION +# ============================================================================= + +DATA_DIR = '/data/dbus-windy-station' +CONFIG_FILE = DATA_DIR + '/station_config.json' + +# ============================================================================= +# LOGGING CONFIGURATION +# ============================================================================= + +LOGGING_CONFIG = { + 'level': 'INFO', + 'console': True, + 'include_timestamp': False, +} diff --git a/dbus-windy-station/gui-v2-source/WindyStation_PageSettings.qml b/dbus-windy-station/gui-v2-source/WindyStation_PageSettings.qml new file mode 100644 index 0000000..433c8a4 --- /dev/null +++ b/dbus-windy-station/gui-v2-source/WindyStation_PageSettings.qml @@ -0,0 +1,187 @@ +import QtQuick +import Victron.VenusOS + +Page { + id: root + title: "Windy Station" + + property string bindPrefix: "dbus/com.victronenergy.windystation" + + VeQuickItem { id: connectedItem; uid: bindPrefix + "/Connected" } + VeQuickItem { id: windSpeedItem; uid: bindPrefix + "/WindSpeed" } + VeQuickItem { id: windGustItem; uid: bindPrefix + "/WindGust" } + VeQuickItem { id: windDirItem; uid: bindPrefix + "/WindDirection" } + VeQuickItem { id: temperatureItem; uid: bindPrefix + "/Temperature" } + VeQuickItem { id: pressureItem; uid: bindPrefix + "/Pressure" } + VeQuickItem { id: lastReportItem; uid: bindPrefix + "/LastReportTimeAgo" } + VeQuickItem { id: unitsItem; uid: bindPrefix + "/Settings/Units" } + VeQuickItem { id: enabledItem; uid: bindPrefix + "/Settings/Enabled" } + VeQuickItem { id: shareItem; uid: bindPrefix + "/Settings/ShareOption" } + + property bool imperial: unitsItem.isValid && unitsItem.value === 1 + property bool serviceConnected: connectedItem.isValid + + function fmtWind(ms) { + if (ms === undefined || ms === null || isNaN(ms)) return "--" + if (imperial) return (ms * 1.94384).toFixed(1) + return ms.toFixed(1) + } + + function fmtTemp(c) { + if (c === undefined || c === null || isNaN(c)) return "--" + if (imperial) return (c * 9.0 / 5.0 + 32.0).toFixed(1) + return c.toFixed(1) + } + + function fmtPressure(hpa) { + if (hpa === undefined || hpa === null || isNaN(hpa)) return "--" + if (imperial) return (hpa * 0.02953).toFixed(2) + return hpa.toFixed(1) + } + + property string windUnit: imperial ? "kts" : "m/s" + property string tempUnit: imperial ? "°F" : "°C" + property string pressUnit: imperial ? "inHg" : "hPa" + + GradientListView { + model: VisibleItemModel { + + PrimaryListLabel { + text: "Service not running - check installation" + preferredVisible: !serviceConnected + } + + ListText { + text: "Wind" + secondaryText: "%1 %2 @ %3° Gust: %4 %5" + .arg(fmtWind(windSpeedItem.isValid ? windSpeedItem.value : null)) + .arg(windUnit) + .arg(windDirItem.isValid && windDirItem.value !== undefined + ? Math.round(windDirItem.value) : "--") + .arg(fmtWind(windGustItem.isValid ? windGustItem.value : null)) + .arg(windUnit) + preferredVisible: serviceConnected + } + + ListText { + text: "Temperature / Pressure" + secondaryText: "%1 %2 | %3 %4" + .arg(fmtTemp(temperatureItem.isValid ? temperatureItem.value : null)) + .arg(tempUnit) + .arg(fmtPressure(pressureItem.isValid ? pressureItem.value : null)) + .arg(pressUnit) + preferredVisible: serviceConnected + } + + ListText { + text: "Last reported" + secondaryText: lastReportItem.isValid ? lastReportItem.value : "--" + preferredVisible: serviceConnected + } + + ListSwitch { + text: "Enable uploads" + dataItem.uid: root.bindPrefix + "/Settings/Enabled" + preferredVisible: serviceConnected + } + + ListSwitch { + text: "Imperial units (kts/°F/inHg)" + dataItem.uid: root.bindPrefix + "/Settings/Units" + preferredVisible: serviceConnected + } + + ListTextField { + text: "API Key" + dataItem.uid: root.bindPrefix + "/Settings/ApiKey" + placeholderText: "Enter API key" + preferredVisible: serviceConnected + } + + ListTextField { + text: "Station ID" + dataItem.uid: root.bindPrefix + "/Settings/StationId" + placeholderText: "Enter station ID" + preferredVisible: serviceConnected + } + + ListTextField { + text: "Station Password" + dataItem.uid: root.bindPrefix + "/Settings/StationPassword" + placeholderText: "Enter password" + preferredVisible: serviceConnected + } + + ListTextField { + text: "Station Name" + dataItem.uid: root.bindPrefix + "/Settings/Name" + placeholderText: "Enter station name" + preferredVisible: serviceConnected + } + + ListRadioButtonGroup { + text: "Share option" + optionModel: [ + { display: "Public", value: 0 }, + { display: "Only Windy", value: 1 }, + { display: "Private", value: 2 } + ] + currentIndex: { + if (!shareItem.isValid || !shareItem.value) return 0 + if (shareItem.value === "only_windy") return 1 + if (shareItem.value === "private") return 2 + return 0 + } + onOptionClicked: function(index) { + currentIndex = index + var values = ["public", "only_windy", "private"] + shareItem.setValue(values[index]) + } + preferredVisible: serviceConnected + } + + ListTextField { + text: "Station Type" + dataItem.uid: root.bindPrefix + "/Settings/StationType" + placeholderText: "e.g. Boat, Airmar WX200" + preferredVisible: serviceConnected + } + + ListSpinBox { + text: "Upload interval (s)" + dataItem.uid: root.bindPrefix + "/Settings/UploadInterval" + from: 300 + to: 900 + stepSize: 60 + preferredVisible: serviceConnected && enabledItem.isValid && enabledItem.value === 1 + } + + ListSpinBox { + text: "Elevation (m)" + dataItem.uid: root.bindPrefix + "/Settings/ElevM" + from: 0 + to: 1000 + stepSize: 1 + preferredVisible: serviceConnected + } + + ListSpinBox { + text: "Wind sensor height AGL (m)" + dataItem.uid: root.bindPrefix + "/Settings/AglWind" + from: 0 + to: 100 + stepSize: 1 + preferredVisible: serviceConnected + } + + ListSpinBox { + text: "Temp sensor height AGL (m)" + dataItem.uid: root.bindPrefix + "/Settings/AglTemp" + from: 0 + to: 100 + stepSize: 1 + preferredVisible: serviceConnected + } + } + } +} diff --git a/dbus-windy-station/install.sh b/dbus-windy-station/install.sh new file mode 100755 index 0000000..2b5fda9 --- /dev/null +++ b/dbus-windy-station/install.sh @@ -0,0 +1,178 @@ +#!/bin/bash +# +# Installation script for Windy Station on Venus OS +# +# Run this on the Venus OS device after copying files to /data/dbus-windy-station/ +# +# Usage: +# chmod +x install.sh +# ./install.sh +# + +set -e + +INSTALL_DIR="/data/dbus-windy-station" + +# Find velib_python +VELIB_DIR="" +if [ -d "/opt/victronenergy/velib_python" ]; then + VELIB_DIR="/opt/victronenergy/velib_python" +else + for candidate in \ + "/opt/victronenergy/dbus-systemcalc-py/ext/velib_python" \ + "/opt/victronenergy/dbus-generator/ext/velib_python" \ + "/opt/victronenergy/dbus-mqtt/ext/velib_python" \ + "/opt/victronenergy/dbus-digitalinputs/ext/velib_python" \ + "/opt/victronenergy/vrmlogger/ext/velib_python" + do + if [ -d "$candidate" ] && [ -f "$candidate/vedbus.py" ]; then + VELIB_DIR="$candidate" + break + fi + done +fi + +if [ -z "$VELIB_DIR" ]; then + VEDBUS_PATH=$(find /opt/victronenergy -name "vedbus.py" -path "*/velib_python/*" 2>/dev/null | head -1) + if [ -n "$VEDBUS_PATH" ]; then + VELIB_DIR=$(dirname "$VEDBUS_PATH") + fi +fi + +# Determine service directory +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +elif [ -L "/service" ]; then + SERVICE_DIR=$(readlink -f /service) +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "==================================================" +echo "Windy Station - Installation" +echo "==================================================" + +if [ ! -d "$SERVICE_DIR" ]; then + echo "ERROR: This doesn't appear to be a Venus OS device." + echo " Service directory not found." + exit 1 +fi + +echo "Detected service directory: $SERVICE_DIR" + +if [ ! -f "$INSTALL_DIR/windy_station.py" ]; then + echo "ERROR: Installation files not found in $INSTALL_DIR" + echo " Please copy all files to $INSTALL_DIR first." + exit 1 +fi +if [ ! -f "$INSTALL_DIR/service/run" ]; then + echo "ERROR: service/run not found. The package is incomplete." + exit 1 +fi + +echo "1. Making scripts executable..." +chmod +x "$INSTALL_DIR/service/run" +chmod +x "$INSTALL_DIR/service/log/run" +chmod +x "$INSTALL_DIR/windy_station.py" +if [ -f "$INSTALL_DIR/install_gui.sh" ]; then + chmod +x "$INSTALL_DIR/install_gui.sh" +fi + +echo "2. Creating velib_python symlink..." +if [ -z "$VELIB_DIR" ]; then + echo "ERROR: Could not find velib_python on this system." + exit 1 +fi +echo " Found velib_python at: $VELIB_DIR" +mkdir -p "$INSTALL_DIR/ext" +if [ -L "$INSTALL_DIR/ext/velib_python" ]; then + CURRENT_TARGET=$(readlink "$INSTALL_DIR/ext/velib_python") + if [ "$CURRENT_TARGET" != "$VELIB_DIR" ]; then + echo " Updating symlink (was: $CURRENT_TARGET)" + rm "$INSTALL_DIR/ext/velib_python" + fi +fi +if [ ! -L "$INSTALL_DIR/ext/velib_python" ]; then + ln -s "$VELIB_DIR" "$INSTALL_DIR/ext/velib_python" + echo " Symlink created: $INSTALL_DIR/ext/velib_python -> $VELIB_DIR" +else + echo " Symlink already exists" +fi + +echo "3. Creating service symlink..." +if [ -L "$SERVICE_DIR/dbus-windy-station" ]; then + echo " Service link already exists, removing old link..." + rm "$SERVICE_DIR/dbus-windy-station" +fi +if [ -e "$SERVICE_DIR/dbus-windy-station" ]; then + rm -rf "$SERVICE_DIR/dbus-windy-station" +fi +ln -s "$INSTALL_DIR/service" "$SERVICE_DIR/dbus-windy-station" + +if [ -L "$SERVICE_DIR/dbus-windy-station" ]; then + echo " Symlink created: $SERVICE_DIR/dbus-windy-station -> $INSTALL_DIR/service" +else + echo "ERROR: Failed to create service symlink" + exit 1 +fi + +echo "4. Creating log directory..." +mkdir -p /var/log/dbus-windy-station + +echo "5. Setting up rc.local for persistence..." +RC_LOCAL="/data/rc.local" +if [ ! -f "$RC_LOCAL" ]; then + echo "#!/bin/bash" > "$RC_LOCAL" + chmod +x "$RC_LOCAL" +fi + +if ! grep -q "dbus-windy-station" "$RC_LOCAL"; then + echo "" >> "$RC_LOCAL" + echo "# Windy Station" >> "$RC_LOCAL" + echo "if [ ! -L $SERVICE_DIR/dbus-windy-station ]; then" >> "$RC_LOCAL" + echo " ln -s /data/dbus-windy-station/service $SERVICE_DIR/dbus-windy-station" >> "$RC_LOCAL" + echo "fi" >> "$RC_LOCAL" + echo " Added to rc.local for persistence across firmware updates" +else + echo " Already in rc.local" +fi + +echo "6. Activating service..." +sleep 2 +if command -v svstat >/dev/null 2>&1; then + if svstat "$SERVICE_DIR/dbus-windy-station" 2>/dev/null | grep -q "up"; then + echo " Service is running" + else + echo " Waiting for service to start..." + sleep 3 + fi +fi + +echo "" +echo "==================================================" +echo "Installation complete!" +echo "==================================================" +echo "" + +if command -v svstat >/dev/null 2>&1; then + echo "Current status:" + svstat "$SERVICE_DIR/dbus-windy-station" 2>/dev/null || echo " Service not yet detected by svscan" + echo "" +fi + +echo "IMPORTANT: Configure via Settings menu:" +echo " 1. Run: ./install_gui.sh (adds Windy Station to Settings)" +echo " 2. Open Settings > Windy Station and enter your credentials" +echo " 3. Or use MQTT: N//windystation/0/Settings/..." +echo "" +echo "To check status:" +echo " svstat $SERVICE_DIR/dbus-windy-station" +echo "" +echo "To view logs:" +echo " tail -F /var/log/dbus-windy-station/current | tai64nlocal" +echo "" +echo "Install GUI integration (required for configuration):" +echo " ./install_gui.sh" +echo "" diff --git a/dbus-windy-station/install_gui.sh b/dbus-windy-station/install_gui.sh new file mode 100755 index 0000000..360536e --- /dev/null +++ b/dbus-windy-station/install_gui.sh @@ -0,0 +1,164 @@ +#!/bin/bash +# +# Install GUI modifications for Windy Station +# +# This script: +# 1. Copies PageSettingsWindyStation.qml to the GUI directory +# 2. Patches PageSettings.qml to add Windy Station menu entry +# 3. Restarts the GUI +# +# Note: These changes will be lost on firmware update. Run again after updating. +# +# Usage: +# ./install_gui.sh # Install GUI modifications +# ./install_gui.sh --remove # Remove GUI modifications and restore originals +# + +set -e + +QML_DIR="/opt/victronenergy/gui/qml" +INSTALL_DIR="/data/dbus-windy-station" +BACKUP_DIR="/data/dbus-windy-station/backup" + +install_gui() { + echo "==================================================" + echo "Installing Windy Station GUI modifications..." + echo "==================================================" + + if [ ! -f "$INSTALL_DIR/windy_station.py" ]; then + echo "ERROR: Main service not installed. Run install.sh first." + exit 1 + fi + + if [ ! -d "$QML_DIR" ]; then + echo "ERROR: GUI directory not found: $QML_DIR" + exit 1 + fi + + mkdir -p "$BACKUP_DIR" + + # Step 1: Copy our QML file + echo "1. Installing PageSettingsWindyStation.qml..." + if [ -f "$INSTALL_DIR/qml/PageSettingsWindyStation.qml" ]; then + cp "$INSTALL_DIR/qml/PageSettingsWindyStation.qml" "$QML_DIR/" + echo " Copied to $QML_DIR/" + else + echo "ERROR: PageSettingsWindyStation.qml not found" + exit 1 + fi + + # Step 2: Patch PageSettings.qml (insert before "Services" entry) + echo "2. Patching PageSettings.qml..." + SETTINGS_QML="$QML_DIR/PageSettings.qml" + if [ ! -f "$SETTINGS_QML" ]; then + echo " PageSettings.qml not found - manual GUI configuration needed" + elif grep -q "PageSettingsWindyStation" "$SETTINGS_QML"; then + echo " Already patched, skipping..." + else + if [ ! -f "$BACKUP_DIR/PageSettings.qml.orig" ]; then + cp "$SETTINGS_QML" "$BACKUP_DIR/PageSettings.qml.orig" + echo " Backup saved to $BACKUP_DIR/" + fi + python3 << 'PYTHON_SCRIPT' +import re +qml_file = "/opt/victronenergy/gui/qml/PageSettings.qml" +with open(qml_file, 'r') as f: + content = f.read() +if 'PageSettingsWindyStation' in content: + print("Already patched") + exit(0) +menu_entry = ''' +\t\tMbSubMenu { +\t\t\tdescription: qsTr("Windy Station") +\t\t\tsubpage: +\t\t\t\tComponent { +\t\t\t\tPageSettingsWindyStation { +\t\t\t\t\ttitle: qsTr("Windy Station") +\t\t\t\t} +\t\t\t} +\t\t} + +''' +# Try inserting before "Services" menu entry +pattern = r'(\n\t\tMbSubMenu \{\n\t\t\tdescription: qsTr\("Services"\))' +new_content = re.sub(pattern, menu_entry + r'\1', content, count=1) +if new_content == content: + # Fallback: insert before "Debug" entry + pattern2 = r'(\n\t\tMbSubMenu \{\n\t\t\tdescription: qsTr\("Debug"\))' + new_content = re.sub(pattern2, menu_entry + r'\1', content, count=1) +if new_content == content: + # Last resort: insert before the closing brace of the model + pattern3 = r'(\n\t\}\n\})' + new_content = re.sub(pattern3, menu_entry + r'\1', content, count=1) +if new_content == content: + print("ERROR: Could not find insertion point in PageSettings.qml") + exit(1) +with open(qml_file, 'w') as f: + f.write(new_content) +print("Patched successfully") +PYTHON_SCRIPT + if [ $? -ne 0 ]; then exit 1; fi + echo " Patch applied" + fi + + # Step 3: Restart GUI + echo "3. Restarting GUI..." + if [ -d /service/start-gui ]; then + svc -t /service/start-gui + sleep 2 + elif [ -d /service/gui ]; then + svc -t /service/gui + sleep 2 + else + echo " Note: GUI service not found. Restart manually or reboot." + fi + + echo "" + echo "==================================================" + echo "GUI installation complete!" + echo "==================================================" + echo "" + echo "Windy Station should appear in: Settings -> Windy Station" + echo "If GUI goes blank, run: install_gui.sh --remove" + echo "" + echo "Note: Run this script again after firmware updates." + echo "" +} + +remove_gui() { + echo "==================================================" + echo "Removing Windy Station GUI modifications..." + echo "==================================================" + + if [ -f "$QML_DIR/PageSettingsWindyStation.qml" ]; then + rm "$QML_DIR/PageSettingsWindyStation.qml" + echo "1. Removed PageSettingsWindyStation.qml" + fi + + if [ -f "$BACKUP_DIR/PageSettings.qml.orig" ]; then + cp "$BACKUP_DIR/PageSettings.qml.orig" "$QML_DIR/PageSettings.qml" + echo "2. Restored original PageSettings.qml" + fi + + echo "3. Restarting GUI..." + if [ -d /service/start-gui ]; then + svc -t /service/start-gui + sleep 2 + elif [ -d /service/gui ]; then + svc -t /service/gui + sleep 2 + fi + + echo "" + echo "GUI modifications removed." + echo "" +} + +case "$1" in + --remove) + remove_gui + ;; + *) + install_gui + ;; +esac diff --git a/dbus-windy-station/install_gui_v2.sh b/dbus-windy-station/install_gui_v2.sh new file mode 100644 index 0000000..d6f445c --- /dev/null +++ b/dbus-windy-station/install_gui_v2.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# +# Install GUI v2 plugin for Windy Station +# +# Uses the Venus OS gui-v2 plugin system (requires Venus OS v3.70~45+). +# The plugin appears under Settings -> Integrations -> UI Plugins -> Windy Station. +# +# Unlike the v1 GUI installer, this does NOT patch any system files and +# survives firmware updates automatically. +# +# Usage: +# ./install_gui_v2.sh # Install GUI v2 plugin +# ./install_gui_v2.sh --remove # Remove GUI v2 plugin +# + +set -e + +INSTALL_DIR="/data/dbus-windy-station" +APP_NAME="dbus-windy-station" +PLUGIN_NAME="WindyStation" +APP_DIR="/data/apps/available/$APP_NAME" +ENABLED_DIR="/data/apps/enabled" +COMPILER="$INSTALL_DIR/compile_gui_v2_plugin.py" +QML_SOURCE="gui-v2-source/${PLUGIN_NAME}_PageSettings.qml" +MIN_GUI_VERSION="v1.2.13" + +install_plugin() { + echo "==================================================" + echo "Installing Windy Station GUI v2 plugin..." + echo "==================================================" + + if [ ! -f "$INSTALL_DIR/windy_station.py" ]; then + echo "ERROR: Main service not installed. Run install.sh first." + exit 1 + fi + + if [ ! -f "$COMPILER" ]; then + echo "ERROR: Plugin compiler not found at $COMPILER" + exit 1 + fi + + if [ ! -f "$INSTALL_DIR/$QML_SOURCE" ]; then + echo "ERROR: QML source not found: $INSTALL_DIR/$QML_SOURCE" + exit 1 + fi + + echo "1. Creating app directory structure..." + mkdir -p "$APP_DIR/gui-v2-source" + mkdir -p "$APP_DIR/gui-v2" + + echo "2. Copying QML source and compiler..." + cp "$INSTALL_DIR/$QML_SOURCE" "$APP_DIR/gui-v2-source/" + cp "$COMPILER" "$APP_DIR/gui-v2-source/" + + echo "3. Compiling plugin descriptor..." + cd "$APP_DIR/gui-v2-source" + python3 compile_gui_v2_plugin.py \ + --name "$PLUGIN_NAME" \ + --min-required-version "$MIN_GUI_VERSION" \ + --settings "${PLUGIN_NAME}_PageSettings.qml" + + if [ ! -f "${PLUGIN_NAME}.json" ]; then + echo "ERROR: Plugin compilation failed - ${PLUGIN_NAME}.json not generated" + exit 1 + fi + + echo "4. Installing plugin descriptor..." + cp "${PLUGIN_NAME}.json" "$APP_DIR/gui-v2/" + + echo "5. Enabling app..." + mkdir -p "$ENABLED_DIR" + if [ -L "$ENABLED_DIR/$APP_NAME" ]; then + rm "$ENABLED_DIR/$APP_NAME" + fi + ln -s "$APP_DIR" "$ENABLED_DIR/$APP_NAME" + + echo "6. Restarting GUI..." + if [ -d /service/start-gui ]; then + svc -t /service/start-gui + sleep 2 + fi + + echo "" + echo "==================================================" + echo "GUI v2 plugin installed!" + echo "==================================================" + echo "" + echo "Navigate to: Settings -> Integrations -> UI Plugins -> Windy Station" + echo "" +} + +remove_plugin() { + echo "==================================================" + echo "Removing Windy Station GUI v2 plugin..." + echo "==================================================" + + if [ -L "$ENABLED_DIR/$APP_NAME" ]; then + rm "$ENABLED_DIR/$APP_NAME" + echo "1. Removed app symlink" + else + echo "1. App symlink not found (already removed)" + fi + + if [ -d "$APP_DIR" ]; then + rm -rf "$APP_DIR" + echo "2. Removed app directory" + else + echo "2. App directory not found (already removed)" + fi + + echo "3. Restarting GUI..." + if [ -d /service/start-gui ]; then + svc -t /service/start-gui + sleep 2 + fi + + echo "" + echo "GUI v2 plugin removed." + echo "" +} + +case "$1" in + --remove) + remove_plugin + ;; + *) + install_plugin + ;; +esac diff --git a/dbus-windy-station/qml/PageSettingsWindyStation.qml b/dbus-windy-station/qml/PageSettingsWindyStation.qml new file mode 100644 index 0000000..1d129a8 --- /dev/null +++ b/dbus-windy-station/qml/PageSettingsWindyStation.qml @@ -0,0 +1,193 @@ +import QtQuick 2 +import com.victron.velib 1.0 +import "utils.js" as Utils + +MbPage { + id: root + title: qsTr("Windy Station") + property string bindPrefix: "com.victronenergy.windystation" + property VBusItem connectedItem: VBusItem { bind: Utils.path(bindPrefix, "/Connected") } + property VBusItem windSpeedItem: VBusItem { bind: Utils.path(bindPrefix, "/WindSpeed") } + property VBusItem windGustItem: VBusItem { bind: Utils.path(bindPrefix, "/WindGust") } + property VBusItem windDirItem: VBusItem { bind: Utils.path(bindPrefix, "/WindDirection") } + property VBusItem temperatureItem: VBusItem { bind: Utils.path(bindPrefix, "/Temperature") } + property VBusItem pressureItem: VBusItem { bind: Utils.path(bindPrefix, "/Pressure") } + property VBusItem lastReportItem: VBusItem { bind: Utils.path(bindPrefix, "/LastReportTimeAgo") } + property VBusItem unitsItem: VBusItem { bind: Utils.path(bindPrefix, "/Settings/Units") } + + property bool imperial: unitsItem.valid && unitsItem.value === 1 + + function truncate(val, maxLen) { + if (!val || val.length <= maxLen) return val || "" + return "..." + val.slice(-maxLen) + } + + function fmtWind(ms) { + if (ms === undefined || ms === null) return "--" + if (imperial) return (ms * 1.94384).toFixed(1) + return ms.toFixed(1) + } + + function fmtTemp(c) { + if (c === undefined || c === null) return "--" + if (imperial) return (c * 9.0 / 5.0 + 32.0).toFixed(1) + return c.toFixed(1) + } + + function fmtPressure(hpa) { + if (hpa === undefined || hpa === null) return "--" + if (imperial) return (hpa * 0.02953).toFixed(2) + return hpa.toFixed(1) + } + + property string windUnit: imperial ? "kts" : "m/s" + property string tempUnit: imperial ? "°F" : "°C" + property string pressUnit: imperial ? "inHg" : "hPa" + + model: VisibleItemModel { + MbItemText { + text: qsTr("Service not running - check installation") + show: !connectedItem.valid + } + + MbItemText { + text: qsTr("Wind: %1 %2 @ %3° Gust: %4 %5").arg( + fmtWind(windSpeedItem.valid ? windSpeedItem.value : null)).arg(windUnit).arg( + windDirItem.valid && windDirItem.value !== undefined ? Math.round(windDirItem.value) : "--").arg( + fmtWind(windGustItem.valid ? windGustItem.value : null)).arg(windUnit) + show: connectedItem.valid + } + + MbItemText { + text: qsTr("Temp: %1 %2 Pressure: %3 %4").arg( + fmtTemp(temperatureItem.valid ? temperatureItem.value : null)).arg(tempUnit).arg( + fmtPressure(pressureItem.valid ? pressureItem.value : null)).arg(pressUnit) + show: connectedItem.valid + } + + MbItemText { + text: qsTr("Last reported: %1").arg( + lastReportItem.valid ? lastReportItem.value : "--") + show: connectedItem.valid + } + + MbSwitch { + id: enableSwitch + name: qsTr("Enable uploads") + bind: Utils.path(bindPrefix, "/Settings/Enabled") + enabled: connectedItem.valid + show: connectedItem.valid + } + + MbSwitch { + name: qsTr("Imperial units (kts/°F/inHg)") + bind: Utils.path(bindPrefix, "/Settings/Units") + enabled: connectedItem.valid + show: connectedItem.valid + } + + MbEditBox { + description: qsTr("API Key") + show: connectedItem.valid + item { + bind: Utils.path(bindPrefix, "/Settings/ApiKey") + text: item.valid && item.value ? truncate(item.value, 12) : "not set" + } + function getEditText() { return item.value || "" } + } + + MbEditBox { + description: qsTr("Station ID") + show: connectedItem.valid + item { + bind: Utils.path(bindPrefix, "/Settings/StationId") + } + } + + MbEditBox { + description: qsTr("Station Password") + show: connectedItem.valid + item { + bind: Utils.path(bindPrefix, "/Settings/StationPassword") + text: item.valid && item.value ? truncate(item.value, 12) : "not set" + } + function getEditText() { return item.value || "" } + } + + MbEditBox { + description: qsTr("Station Name") + show: connectedItem.valid + item { + bind: Utils.path(bindPrefix, "/Settings/Name") + } + } + + MbEditBox { + description: qsTr("Share (public/only_windy/private)") + show: connectedItem.valid + item { + bind: Utils.path(bindPrefix, "/Settings/ShareOption") + } + } + + MbEditBox { + description: qsTr("Station Type") + show: connectedItem.valid + item { + bind: Utils.path(bindPrefix, "/Settings/StationType") + } + } + + MbSpinBox { + description: qsTr("Upload interval") + show: connectedItem.valid && enableSwitch.checked + item { + bind: Utils.path(bindPrefix, "/Settings/UploadInterval") + unit: "s" + decimals: 0 + step: 60 + min: 300 + max: 900 + } + } + + MbSpinBox { + description: qsTr("Elevation") + show: connectedItem.valid + item { + bind: Utils.path(bindPrefix, "/Settings/ElevM") + unit: "m" + decimals: 0 + step: 1 + min: 0 + max: 1000 + } + } + + MbSpinBox { + description: qsTr("Wind sensor height (AGL)") + show: connectedItem.valid + item { + bind: Utils.path(bindPrefix, "/Settings/AglWind") + unit: "m" + decimals: 0 + step: 1 + min: 0 + max: 100 + } + } + + MbSpinBox { + description: qsTr("Temp sensor height (AGL)") + show: connectedItem.valid + item { + bind: Utils.path(bindPrefix, "/Settings/AglTemp") + unit: "m" + decimals: 0 + step: 1 + min: 0 + max: 100 + } + } + } +} diff --git a/dbus-windy-station/service/log/run b/dbus-windy-station/service/log/run new file mode 100755 index 0000000..dc25305 --- /dev/null +++ b/dbus-windy-station/service/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec multilog t s99999 n8 /var/log/dbus-windy-station diff --git a/dbus-windy-station/service/run b/dbus-windy-station/service/run new file mode 100755 index 0000000..f64ae3b --- /dev/null +++ b/dbus-windy-station/service/run @@ -0,0 +1,5 @@ +#!/bin/sh +exec 2>&1 +cd /data/dbus-windy-station +export PYTHONPATH="/data/dbus-windy-station/ext/velib_python:$PYTHONPATH" +exec python3 /data/dbus-windy-station/windy_station.py diff --git a/dbus-windy-station/station_config.example.json b/dbus-windy-station/station_config.example.json new file mode 100644 index 0000000..cff4c3a --- /dev/null +++ b/dbus-windy-station/station_config.example.json @@ -0,0 +1,15 @@ +{ + "api_key": "YOUR_WINDY_API_KEY", + "station_id": "YOUR_STATION_ID", + "station_password": "YOUR_STATION_PASSWORD", + "update_interval": 300, + "name": "My Boat", + "share_option": "public", + "station_type": "Airmar WX200", + "operator_text": "", + "operator_url": "", + "operator_logo": null, + "elev_m": 0, + "agl_wind": 5, + "agl_temp": 5 +} diff --git a/dbus-windy-station/uninstall.sh b/dbus-windy-station/uninstall.sh new file mode 100755 index 0000000..6cb5065 --- /dev/null +++ b/dbus-windy-station/uninstall.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Uninstall Windy Station for Venus OS + +INSTALL_DIR="/data/dbus-windy-station" +SERVICE_LINK="dbus-windy-station" + +# Find service directory +if [ -d "/service" ] && [ ! -L "/service" ]; then + SERVICE_DIR="/service" +elif [ -d "/opt/victronenergy/service" ]; then + SERVICE_DIR="/opt/victronenergy/service" +else + SERVICE_DIR="/opt/victronenergy/service" +fi + +echo "Uninstalling Windy Station..." + +# Remove GUI modifications first +if [ -f "$INSTALL_DIR/install_gui.sh" ]; then + bash "$INSTALL_DIR/install_gui.sh" --remove +fi + +# Stop and remove service +if [ -L "$SERVICE_DIR/$SERVICE_LINK" ] || [ -e "$SERVICE_DIR/$SERVICE_LINK" ]; then + echo "Stopping and removing service..." + svc -d "$SERVICE_DIR/$SERVICE_LINK" 2>/dev/null || true + rm -f "$SERVICE_DIR/$SERVICE_LINK" + rm -rf "$SERVICE_DIR/$SERVICE_LINK" +fi + +echo "Service removed. Config and data in $INSTALL_DIR are preserved." +echo "To remove everything: rm -rf $INSTALL_DIR /var/log/dbus-windy-station" diff --git a/dbus-windy-station/windy_station.py b/dbus-windy-station/windy_station.py new file mode 100755 index 0000000..e986f5b --- /dev/null +++ b/dbus-windy-station/windy_station.py @@ -0,0 +1,999 @@ +#!/usr/bin/env python3 +""" +Windy Station for Venus OS + +Reads weather and GPS data from Venus OS D-Bus (published by Raymarine meteo +service) and uploads observations to Windy.com's weather station network. + +Two types of API calls (matching the proven axiom-nmea windy_station example): +1. Station update (PUT /api/v2/pws/{id}): Updates GPS location and metadata + - On first GPS fix, every 60 minutes, or if GPS moves > 200 feet +2. Observation update (GET /api/v2/observation/update): Uploads weather data + - Every 5 minutes (configurable, min 300s) + +Wind speed is reported as a 5-minute running average. +Wind gust is the peak 3-consecutive-sample average over a 10-minute window. + +Settings are configurable via the Venus OS GUI (Settings -> Windy Station) +and persist across restarts via Venus localsettings. +""" + +import json +import logging +import math +import os +import signal +import sys +import time +from collections import deque +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'ext', 'velib_python')) +sys.path.insert(1, '/opt/victronenergy/velib_python') + +try: + from gi.repository import GLib +except ImportError: + print("ERROR: GLib not available. This script must run on Venus OS.") + sys.exit(1) + +try: + import dbus + from dbus.mainloop.glib import DBusGMainLoop + from vedbus import VeDbusService + from settingsdevice import SettingsDevice +except ImportError as e: + print(f"ERROR: Required module not available: {e}") + print("This script must run on Venus OS.") + sys.exit(1) + +from config import ( + SERVICE_NAME, API_BASE, CONFIG_FILE, + OBSERVATION_MIN_INTERVAL, STATION_UPDATE_INTERVAL, + GPS_MOVEMENT_THRESHOLD_METERS, + WIND_AVG_WINDOW, GUST_WINDOW, WIND_DIR_HISTORY_WINDOW, + LOGGING_CONFIG, +) + +VERSION = '1.0.0' + +BUS_ITEM = "com.victronenergy.BusItem" +SYSTEM_SERVICE = "com.victronenergy.system" + + +def _unwrap(v): + """Minimal dbus value unwrap.""" + if v is None: + return None + if isinstance(v, (dbus.Int16, dbus.Int32, dbus.Int64, + dbus.UInt16, dbus.UInt32, dbus.UInt64, dbus.Byte)): + return int(v) + if isinstance(v, dbus.Double): + return float(v) + if isinstance(v, (dbus.String, dbus.Signature)): + return str(v) + if isinstance(v, dbus.Boolean): + return bool(v) + if isinstance(v, dbus.Array): + return [_unwrap(x) for x in v] if len(v) > 0 else None + if isinstance(v, (dbus.Dictionary, dict)): + return {k: _unwrap(x) for k, x in v.items()} + return v + + +def haversine_distance(lat1, lon1, lat2, lon2): + """Great-circle distance in meters between two GPS coordinates.""" + R = 6371000 + phi1, phi2 = math.radians(lat1), math.radians(lat2) + dphi = math.radians(lat2 - lat1) + dlam = math.radians(lon2 - lon1) + a = (math.sin(dphi / 2) ** 2 + + math.cos(phi1) * math.cos(phi2) * math.sin(dlam / 2) ** 2) + return R * 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) + + +# --------------------------------------------------------------------------- +# D-Bus readers for meteo and GPS data +# --------------------------------------------------------------------------- + +class MeteoReader: + """Read weather data from Venus OS D-Bus meteo service.""" + + def __init__(self, bus): + self.bus = bus + self._service = None + self._proxies = {} + + def _get_proxy(self, service, path): + try: + obj = self.bus.get_object(service, path, introspect=False) + return dbus.Interface(obj, BUS_ITEM) + except dbus.exceptions.DBusException: + return None + + def _discover_service(self): + """Find the meteo service on D-Bus.""" + if self._service: + return True + try: + bus_obj = self.bus.get_object('org.freedesktop.DBus', + '/org/freedesktop/DBus') + iface = dbus.Interface(bus_obj, 'org.freedesktop.DBus') + names = iface.ListNames() + for name in names: + name_str = str(name) + if name_str.startswith('com.victronenergy.meteo.'): + self._service = name_str + self._proxies = {} + for p in ('/WindDirection', '/WindSpeed', + '/ExternalTemperature', '/Pressure'): + self._proxies[p] = self._get_proxy(name_str, p) + return True + except dbus.exceptions.DBusException: + pass + return False + + def _read_path(self, path): + if not self._discover_service(): + return None + proxy = self._proxies.get(path) + if proxy is None: + return None + try: + return _unwrap(proxy.GetValue()) + except dbus.exceptions.DBusException: + self._service = None + return None + + def get_wind_direction(self): + """Wind direction in degrees (0-360).""" + return self._read_path('/WindDirection') + + def get_wind_speed(self): + """Wind speed in m/s.""" + return self._read_path('/WindSpeed') + + def get_temperature(self): + """Air temperature in Celsius.""" + return self._read_path('/ExternalTemperature') + + def get_pressure(self): + """Barometric pressure in hPa (mbar).""" + return self._read_path('/Pressure') + + +class GpsReader: + """Read GPS position from Venus OS D-Bus.""" + + def __init__(self, bus): + self.bus = bus + self._gps_service = None + self._proxy_lat = None + self._proxy_lon = None + + def _get_proxy(self, service, path): + try: + obj = self.bus.get_object(service, path, introspect=False) + return dbus.Interface(obj, BUS_ITEM) + except dbus.exceptions.DBusException: + return None + + def _refresh_gps_service(self): + if self._gps_service: + return True + try: + proxy = self._get_proxy(SYSTEM_SERVICE, "/GpsService") + if proxy: + svc = _unwrap(proxy.GetValue()) + if svc and isinstance(svc, str): + self._gps_service = svc + self._proxy_lat = self._get_proxy(svc, "/Position/Latitude") + self._proxy_lon = self._get_proxy(svc, "/Position/Longitude") + if not self._proxy_lat: + self._proxy_lat = self._get_proxy(svc, "/Latitude") + if not self._proxy_lon: + self._proxy_lon = self._get_proxy(svc, "/Longitude") + return True + except dbus.exceptions.DBusException: + pass + return False + + def get_position(self): + """Return (lat, lon) or None if no fix.""" + if not self._refresh_gps_service(): + return None + lat, lon = None, None + try: + if self._proxy_lat: + lat = _unwrap(self._proxy_lat.GetValue()) + if self._proxy_lon: + lon = _unwrap(self._proxy_lon.GetValue()) + except dbus.exceptions.DBusException: + self._gps_service = None + return None + if (lat is not None and lon is not None and + -90 <= float(lat) <= 90 and -180 <= float(lon) <= 180): + return (float(lat), float(lon)) + return None + + +# --------------------------------------------------------------------------- +# Wind calculations +# --------------------------------------------------------------------------- + +class WindSpeedAverager: + """Computes a running average of wind speed over a configurable window.""" + + def __init__(self, window_seconds=300.0): + self.window_seconds = window_seconds + self.samples = deque() + + def add_sample(self, wind_speed_ms): + now = time.time() + self.samples.append((now, wind_speed_ms)) + self._prune(now) + + def _prune(self, now): + cutoff = now - self.window_seconds + while self.samples and self.samples[0][0] < cutoff: + self.samples.popleft() + + def get_average(self): + """Return average wind speed in m/s, or None if no samples.""" + self._prune(time.time()) + if not self.samples: + return None + total = sum(s[1] for s in self.samples) + return total / len(self.samples) + + +class WindGustTracker: + """Tracks wind gust as peak 3-consecutive-sample average over a window.""" + + def __init__(self, window_seconds=600.0): + self.window_seconds = window_seconds + self.samples = deque() + + def add_sample(self, wind_speed_ms): + now = time.time() + self.samples.append((now, wind_speed_ms)) + self._prune(now) + + def _prune(self, now): + cutoff = now - self.window_seconds + while self.samples and self.samples[0][0] < cutoff: + self.samples.popleft() + + def get_gust(self): + """Peak 3-consecutive-sample average in m/s, or None.""" + self._prune(time.time()) + if len(self.samples) < 3: + return None + speeds = [s[1] for s in self.samples] + max_avg = 0.0 + for i in range(len(speeds) - 2): + avg = (speeds[i] + speeds[i + 1] + speeds[i + 2]) / 3.0 + if avg > max_avg: + max_avg = avg + return max_avg if max_avg > 0 else None + + +class WindDirHistory: + """Rolling buffer of wind direction samples for UI trend visualization.""" + + def __init__(self, window_seconds=1800.0): + self.window_seconds = window_seconds + self.samples = deque() + + def add_sample(self, direction_deg, speed_ms=None): + now = time.time() + self.samples.append((now, direction_deg, speed_ms)) + self._prune(now) + + def _prune(self, now): + cutoff = now - self.window_seconds + while self.samples and self.samples[0][0] < cutoff: + self.samples.popleft() + + def to_json(self): + """Serialize to JSON array of {time, dir, spd} with millisecond timestamps.""" + self._prune(time.time()) + out = [] + for entry in self.samples: + ts, d = entry[0], entry[1] + spd = entry[2] if len(entry) > 2 else None + rec = {"time": int(ts * 1000), "dir": round(d, 1)} + if spd is not None: + rec["spd"] = round(spd, 2) + out.append(rec) + return json.dumps(out) + + +# --------------------------------------------------------------------------- +# Main controller +# --------------------------------------------------------------------------- + +class WindyStationController: + """Coordinates D-Bus monitoring, wind calculations, and Windy API uploads.""" + + def __init__(self): + self._setup_logging() + self.logger = logging.getLogger('WindyStation') + self.logger.info(f"Initializing Windy Station v{VERSION}") + + self.bus = dbus.SystemBus() + + self._create_dbus_service() + self._setup_settings() + + self.meteo = MeteoReader(self.bus) + self.gps = GpsReader(self.bus) + + self.wind_averager = WindSpeedAverager(WIND_AVG_WINDOW) + self.gust_tracker = WindGustTracker(GUST_WINDOW) + self.wind_dir_history = WindDirHistory(WIND_DIR_HISTORY_WINDOW) + + # Timing state + self.last_observation_time = 0 + self.last_observation_success = 0 + self.last_station_update = 0 + self.last_station_attempt = 0 + self.last_station_lat = None + self.last_station_lon = None + self.last_wind_sample = 0 + self.last_dir_history_publish = 0 + + GLib.timeout_add(1000, self._main_loop) + self.logger.info("Initialized. Polling every 1s, observations every " + f"{OBSERVATION_MIN_INTERVAL}s") + + def _setup_logging(self): + level = getattr(logging, LOGGING_CONFIG['level'], logging.INFO) + fmt = ('%(asctime)s %(levelname)s %(name)s: %(message)s' + if LOGGING_CONFIG['include_timestamp'] + else '%(levelname)s %(name)s: %(message)s') + logging.basicConfig(level=level, format=fmt, stream=sys.stdout) + + # -- D-Bus service creation ----------------------------------------------- + + def _create_dbus_service(self): + self.logger.info(f"Creating D-Bus service: {SERVICE_NAME}") + + max_retries = 5 + retry_delay = 1.0 + for attempt in range(max_retries): + try: + self.dbus_service = VeDbusService( + SERVICE_NAME, self.bus, register=False) + break + except dbus.exceptions.NameExistsException: + if attempt < max_retries - 1: + self.logger.warning( + f"D-Bus name exists, retrying in {retry_delay}s " + f"(attempt {attempt + 1}/{max_retries})") + time.sleep(retry_delay) + retry_delay *= 2 + else: + raise + + # Management paths + self.dbus_service.add_path('/Mgmt/ProcessName', 'dbus-windy-station') + self.dbus_service.add_path('/Mgmt/ProcessVersion', VERSION) + self.dbus_service.add_path('/Mgmt/Connection', 'local') + + # Device info + self.dbus_service.add_path('/DeviceInstance', 0) + self.dbus_service.add_path('/ProductId', 0xA160) + self.dbus_service.add_path('/ProductName', 'Windy Station') + self.dbus_service.add_path('/FirmwareVersion', VERSION) + self.dbus_service.add_path('/Connected', 1) + + # Status paths (read-only, shown on GUI) + def _status_text(p, v): + labels = {0: 'Idle', 1: 'Active', 2: 'Uploading', 3: 'Error'} + return labels.get(v, 'Unknown') if v is not None else 'Unknown' + + self.dbus_service.add_path('/Status', 0, + gettextcallback=_status_text) + self.dbus_service.add_path('/WindSpeed', None, + gettextcallback=lambda p, v: + f"{v:.1f} m/s" if v is not None else "--") + self.dbus_service.add_path('/WindGust', None, + gettextcallback=lambda p, v: + f"{v:.1f} m/s" if v is not None else "--") + self.dbus_service.add_path('/WindDirection', None, + gettextcallback=lambda p, v: + f"{v:.0f}°" if v is not None else "--") + self.dbus_service.add_path('/Temperature', None, + gettextcallback=lambda p, v: + f"{v:.1f}°C" if v is not None else "--") + self.dbus_service.add_path('/Pressure', None, + gettextcallback=lambda p, v: + f"{v:.1f} hPa" if v is not None else "--") + self.dbus_service.add_path('/LastReportTimeAgo', 'Never') + self.dbus_service.add_path('/LastReportTime', 0, + gettextcallback=self._report_time_text) + self.dbus_service.add_path('/WindDirHistory', '[]') + + # Writable settings + self.dbus_service.add_path('/Settings/Enabled', 1, + writeable=True, + onchangecallback=self._on_setting_changed) + self.dbus_service.add_path('/Settings/ApiKey', '', + writeable=True, + onchangecallback=self._on_setting_changed) + self.dbus_service.add_path('/Settings/StationId', '', + writeable=True, + onchangecallback=self._on_setting_changed) + self.dbus_service.add_path('/Settings/StationPassword', '', + writeable=True, + onchangecallback=self._on_setting_changed) + self.dbus_service.add_path('/Settings/UploadInterval', + OBSERVATION_MIN_INTERVAL, + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: + f"{v}s" if v is not None else "--") + self.dbus_service.add_path('/Settings/Name', '', + writeable=True, + onchangecallback=self._on_setting_changed) + self.dbus_service.add_path('/Settings/ShareOption', 'public', + writeable=True, + onchangecallback=self._on_setting_changed) + self.dbus_service.add_path('/Settings/StationType', '', + writeable=True, + onchangecallback=self._on_setting_changed) + self.dbus_service.add_path('/Settings/ElevM', 0, + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: + f"{v} m" if v is not None else "--") + self.dbus_service.add_path('/Settings/AglWind', 10, + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: + f"{v} m" if v is not None else "--") + self.dbus_service.add_path('/Settings/AglTemp', 2, + writeable=True, + onchangecallback=self._on_setting_changed, + gettextcallback=lambda p, v: + f"{v} m" if v is not None else "--") + self.dbus_service.add_path('/Settings/Units', 0, + writeable=True, + onchangecallback=self._on_setting_changed) + + self.dbus_service.register() + self.logger.info("D-Bus service created") + + # -- Persistent settings -------------------------------------------------- + + def _setup_settings(self): + self.settings = None + try: + path = '/Settings/WindyStation' + settings_def = { + 'Enabled': [path + '/Enabled', 1, 0, 1], + 'ApiKey': [path + '/ApiKey', '', 0, 0], + 'StationId': [path + '/StationId', '', 0, 0], + 'StationPassword': [path + '/StationPassword', '', 0, 0], + 'UploadInterval': [path + '/UploadInterval', + OBSERVATION_MIN_INTERVAL, 300, 900], + 'Name': [path + '/Name', '', 0, 0], + 'ShareOption': [path + '/ShareOption', 'public', 0, 0], + 'StationType': [path + '/StationType', '', 0, 0], + 'ElevM': [path + '/ElevM', 0, 0, 1000], + 'AglWind': [path + '/AglWind', 10, 0, 100], + 'AglTemp': [path + '/AglTemp', 2, 0, 100], + 'Units': [path + '/Units', 0, 0, 1], + } + self.settings = SettingsDevice( + self.bus, settings_def, + self._on_persistent_setting_changed) + if self.settings: + self._load_settings() + self.logger.info("Persistent settings initialized") + except Exception as e: + self.logger.warning(f"Could not initialize persistent settings: {e}") + self._set_defaults() + + self._load_config_file() + + def _set_defaults(self): + self.enabled = True + self.api_key = '' + self.station_id = '' + self.station_password = '' + self.upload_interval = OBSERVATION_MIN_INTERVAL + self.station_name = '' + self.share_option = 'public' + self.station_type = '' + self.elev_m = 0 + self.agl_wind = 10 + self.agl_temp = 2 + self.units = 0 # 0=Metric, 1=Imperial + + def _load_config_file(self): + """Load station_config.json if it exists. Overrides localsettings.""" + try: + if not os.path.exists(CONFIG_FILE): + return + with open(CONFIG_FILE, 'r') as f: + data = json.load(f) + self.logger.info(f"Loading config from {CONFIG_FILE}") + + field_map = { + 'api_key': ('ApiKey', 'api_key', str), + 'apiKey': ('ApiKey', 'api_key', str), + 'station_id': ('StationId', 'station_id', str), + 'stationId': ('StationId', 'station_id', str), + 'id': ('StationId', 'station_id', str), + 'station_password': ('StationPassword', 'station_password', str), + 'stationPassword': ('StationPassword', 'station_password', str), + 'password': ('StationPassword', 'station_password', str), + 'update_interval': ('UploadInterval', 'upload_interval', int), + 'updateInterval': ('UploadInterval', 'upload_interval', int), + 'interval': ('UploadInterval', 'upload_interval', int), + 'name': ('Name', 'station_name', str), + 'share_option': ('ShareOption', 'share_option', str), + 'shareOption': ('ShareOption', 'share_option', str), + 'station_type': ('StationType', 'station_type', str), + 'stationType': ('StationType', 'station_type', str), + 'elev_m': ('ElevM', 'elev_m', float), + 'elevM': ('ElevM', 'elev_m', float), + 'agl_wind': ('AglWind', 'agl_wind', float), + 'aglWind': ('AglWind', 'agl_wind', float), + 'agl_temp': ('AglTemp', 'agl_temp', float), + 'aglTemp': ('AglTemp', 'agl_temp', float), + } + + for json_key, value in data.items(): + if json_key not in field_map: + continue + setting_name, attr_name, cast = field_map[json_key] + try: + typed_value = cast(value) if value is not None else None + if typed_value is None: + continue + setattr(self, attr_name, typed_value) + self._save_setting(setting_name, typed_value) + dbus_path = f'/Settings/{setting_name}' + self.dbus_service[dbus_path] = typed_value + except (ValueError, TypeError): + self.logger.warning( + f"Invalid value for {json_key}: {value}") + + # Normalize share_option + share_map = { + 'public': 'public', 'open': 'public', + 'only_windy': 'only_windy', 'onlywindy': 'only_windy', + 'private': 'private', + } + normalized = share_map.get( + self.share_option.lower().replace(' ', '')) + if normalized: + self.share_option = normalized + + # Enforce interval bounds + self.upload_interval = max(300, min(900, self.upload_interval)) + + self.logger.info( + f"Config file applied: station_id={self.station_id}, " + f"name={self.station_name}") + + except json.JSONDecodeError as e: + self.logger.error(f"Invalid JSON in {CONFIG_FILE}: {e}") + except Exception as e: + self.logger.error(f"Error reading {CONFIG_FILE}: {e}") + + def _load_settings(self): + if not self.settings: + return + try: + self.enabled = bool(self.settings['Enabled']) + self.api_key = str(self.settings['ApiKey'] or '') + self.station_id = str(self.settings['StationId'] or '') + self.station_password = str(self.settings['StationPassword'] or '') + self.upload_interval = max( + 300, min(900, int(self.settings['UploadInterval']))) + self.station_name = str(self.settings['Name'] or '') + self.share_option = str(self.settings['ShareOption'] or 'public') + self.station_type = str(self.settings['StationType'] or '') + self.elev_m = float(self.settings['ElevM'] or 0) + self.agl_wind = float(self.settings['AglWind'] or 10) + self.agl_temp = float(self.settings['AglTemp'] or 2) + self.units = int(self.settings['Units'] or 0) + + self.dbus_service['/Settings/Enabled'] = 1 if self.enabled else 0 + self.dbus_service['/Settings/ApiKey'] = self.api_key + self.dbus_service['/Settings/StationId'] = self.station_id + self.dbus_service['/Settings/StationPassword'] = self.station_password + self.dbus_service['/Settings/UploadInterval'] = self.upload_interval + self.dbus_service['/Settings/Name'] = self.station_name + self.dbus_service['/Settings/ShareOption'] = self.share_option + self.dbus_service['/Settings/StationType'] = self.station_type + self.dbus_service['/Settings/ElevM'] = self.elev_m + self.dbus_service['/Settings/AglWind'] = self.agl_wind + self.dbus_service['/Settings/AglTemp'] = self.agl_temp + self.dbus_service['/Settings/Units'] = self.units + + self.logger.info( + f"Loaded settings: station_id={self.station_id or '(empty)'}, " + f"interval={self.upload_interval}s") + except Exception as e: + self.logger.warning(f"Error loading settings: {e}") + self._set_defaults() + + def _on_persistent_setting_changed(self, setting, old_value, new_value): + self.logger.info(f"Persistent setting changed: {setting} = {new_value}") + self._load_settings() + + def _on_setting_changed(self, path, value): + self.logger.info(f"Setting changed: {path} = {value}") + if path == '/Settings/Enabled': + self.enabled = bool(value) + self._save_setting('Enabled', 1 if self.enabled else 0) + elif path == '/Settings/ApiKey': + self.api_key = str(value) if value else '' + self._save_setting('ApiKey', self.api_key) + elif path == '/Settings/StationId': + self.station_id = str(value) if value else '' + self._save_setting('StationId', self.station_id) + elif path == '/Settings/StationPassword': + self.station_password = str(value) if value else '' + self._save_setting('StationPassword', self.station_password) + elif path == '/Settings/UploadInterval': + val = int(value) if value else OBSERVATION_MIN_INTERVAL + self.upload_interval = max(300, min(900, val)) + self._save_setting('UploadInterval', self.upload_interval) + elif path == '/Settings/Name': + self.station_name = str(value) if value else '' + self._save_setting('Name', self.station_name) + elif path == '/Settings/ShareOption': + self.share_option = str(value) if value else 'public' + self._save_setting('ShareOption', self.share_option) + elif path == '/Settings/StationType': + self.station_type = str(value) if value else '' + self._save_setting('StationType', self.station_type) + elif path == '/Settings/ElevM': + self.elev_m = float(value) if value is not None else 0 + self._save_setting('ElevM', self.elev_m) + elif path == '/Settings/AglWind': + self.agl_wind = float(value) if value is not None else 10 + self._save_setting('AglWind', self.agl_wind) + elif path == '/Settings/AglTemp': + self.agl_temp = float(value) if value is not None else 2 + self._save_setting('AglTemp', self.agl_temp) + elif path == '/Settings/Units': + self.units = int(value) if value is not None else 0 + self._save_setting('Units', self.units) + return True + + def _save_setting(self, name, value): + if self.settings: + try: + self.settings[name] = value + except Exception as e: + self.logger.warning(f"Failed to save setting {name}: {e}") + + # -- Display helpers ------------------------------------------------------ + + def _report_time_text(self, path, value): + try: + if value is None or (isinstance(value, (int, float)) and value <= 0): + return "Never" + diff = time.time() - float(value) + if diff < 60: + return "Just now" + if diff < 3600: + return f"{int(diff / 60)}m ago" + if diff < 86400: + return f"{int(diff / 3600)}h ago" + return time.strftime("%Y-%m-%d %H:%M", time.localtime(float(value))) + except (TypeError, ValueError): + return "Never" + + def _update_report_time_ago(self): + if self.last_observation_success <= 0: + self.dbus_service['/LastReportTimeAgo'] = 'Never' + else: + diff = time.time() - self.last_observation_success + m = int(diff / 60) + h, rm = m // 60, m % 60 + if h > 0: + self.dbus_service['/LastReportTimeAgo'] = f"{h}h {rm}m" + else: + self.dbus_service['/LastReportTimeAgo'] = f"{rm}m" + + # -- Station update (PUT) ------------------------------------------------- + + def _should_update_station(self, lat, lon): + if lat is None or lon is None: + return False + now = time.time() + if self.last_station_attempt > 0 and now - self.last_station_attempt < 60: + return False + if self.last_station_update == 0: + return True + if now - self.last_station_update >= STATION_UPDATE_INTERVAL: + return True + if self.last_station_lat is not None and self.last_station_lon is not None: + dist = haversine_distance( + self.last_station_lat, self.last_station_lon, lat, lon) + if dist >= GPS_MOVEMENT_THRESHOLD_METERS: + return True + return False + + def _update_station(self, lat, lon): + """PUT station metadata and GPS location to Windy API.""" + self.last_station_attempt = time.time() + + if not self.api_key or not self.station_id: + self.logger.warning("Cannot update station: API key or station ID not set") + return False + + url = f"{API_BASE}/pws/{self.station_id}" + payload = { + "share_option": self.share_option or "public", + } + if self.station_type: + payload["station_type"] = self.station_type + if self.station_name: + payload["name"] = self.station_name + if self.elev_m is not None: + payload["elev_m"] = self.elev_m + if self.agl_wind is not None: + payload["agl_wind"] = self.agl_wind + if self.agl_temp is not None: + payload["agl_temp"] = self.agl_temp + if lat is not None and lon is not None: + payload["lat"] = round(lat, 6) + payload["lon"] = round(lon, 6) + + try: + request = Request( + url, + data=json.dumps(payload).encode('utf-8'), + headers={ + "Content-Type": "application/json", + "windy-api-key": self.api_key, + }, + method='PUT' + ) + with urlopen(request, timeout=30) as response: + if response.status == 200: + self.last_station_update = time.time() + if lat is not None and lon is not None: + self.last_station_lat = lat + self.last_station_lon = lon + self.logger.info(f"Station updated: {lat:.6f}, {lon:.6f}") + return True + body = response.read().decode('utf-8') + self.logger.error(f"Station update error {response.status}: {body}") + return False + except HTTPError as e: + body = "" + try: + body = e.read().decode('utf-8') + except Exception: + pass + self.logger.error(f"Station update HTTP {e.code}: {e.reason} {body}") + return False + except (URLError, Exception) as e: + self.logger.error(f"Station update error: {e}") + return False + + # -- Observation upload (GET) --------------------------------------------- + + def _send_observation(self): + """GET observation update to Windy API with query parameters.""" + if not self.station_id or not self.station_password: + self.logger.warning( + "Cannot send observation: station ID or password not set") + return False + + params = { + "id": self.station_id, + "PASSWORD": self.station_password, + } + + has_data = False + + wind_dir = self.meteo.get_wind_direction() + if wind_dir is not None: + params["winddir"] = int(round(wind_dir)) + has_data = True + + avg_wind = self.wind_averager.get_average() + if avg_wind is not None: + params["wind"] = round(avg_wind, 1) + has_data = True + + gust = self.gust_tracker.get_gust() + if gust is not None: + params["gust"] = round(gust, 1) + + temp = self.meteo.get_temperature() + if temp is not None: + params["temp"] = round(temp, 1) + has_data = True + + pressure = self.meteo.get_pressure() + if pressure is not None: + params["pressure"] = round(pressure * 100) # hPa -> Pa + has_data = True + + if not has_data: + self.logger.debug("No weather data to send") + return False + + url = f"{API_BASE}/observation/update?{urlencode(params)}" + + try: + request = Request(url, method='GET') + with urlopen(request, timeout=30) as response: + if response.status == 200: + self.logger.info( + f"Observation sent: wind={params.get('wind', 'N/A')} " + f"gust={params.get('gust', 'N/A')} " + f"temp={params.get('temp', 'N/A')} " + f"pressure={params.get('pressure', 'N/A')}") + return True + body = response.read().decode('utf-8') + self.logger.error( + f"Observation error {response.status}: {body}") + return False + except HTTPError as e: + body = "" + try: + body = e.read().decode('utf-8') + except Exception: + pass + self.logger.error( + f"Observation HTTP {e.code}: {e.reason} {body}") + if e.code == 429: + self._handle_rate_limit(body) + return False + except (URLError, Exception) as e: + self.logger.error(f"Observation error: {e}") + return False + + def _handle_rate_limit(self, body): + """Parse a 429 response and defer the next observation until retry_after.""" + try: + data = json.loads(body) + retry_iso = data.get("retry_after") + if retry_iso: + from datetime import datetime, timezone + retry_dt = datetime.fromisoformat( + retry_iso.replace("Z", "+00:00")) + delay = (retry_dt - datetime.now(timezone.utc)).total_seconds() + if delay > 0: + self.last_observation_time = time.time() + delay + self.logger.warning( + f"Rate-limited by API; deferring next observation " + f"{delay:.0f}s until {retry_iso}") + return + except Exception: + pass + self.last_observation_time = time.time() + 300 + self.logger.warning( + "Rate-limited by API; deferring next observation 300s") + + # -- Main loop ------------------------------------------------------------ + + def _main_loop(self): + try: + if not self.enabled: + self.dbus_service['/Status'] = 0 + self._update_report_time_ago() + return True + + now = time.time() + + # Sample wind every second for averaging and gust tracking + wind_dir = self.meteo.get_wind_direction() + if now - self.last_wind_sample >= 1.0: + wind_ms = self.meteo.get_wind_speed() + if wind_ms is not None: + self.wind_averager.add_sample(wind_ms) + self.gust_tracker.add_sample(wind_ms) + if wind_dir is not None: + self.wind_dir_history.add_sample(wind_dir, wind_ms) + self.last_wind_sample = now + + # Update current conditions on D-Bus (for GUI display) + avg_wind = self.wind_averager.get_average() + self.dbus_service['/WindSpeed'] = ( + round(avg_wind, 1) if avg_wind is not None else None) + gust = self.gust_tracker.get_gust() + self.dbus_service['/WindGust'] = ( + round(gust, 1) if gust is not None else None) + self.dbus_service['/WindDirection'] = wind_dir + self.dbus_service['/Temperature'] = self.meteo.get_temperature() + self.dbus_service['/Pressure'] = self.meteo.get_pressure() + + # Publish direction history (throttled) + if now - self.last_dir_history_publish >= 15.0: + self.dbus_service['/WindDirHistory'] = ( + self.wind_dir_history.to_json()) + self.last_dir_history_publish = now + + # GPS position + pos = self.gps.get_position() + lat = pos[0] if pos else None + lon = pos[1] if pos else None + + # Check credentials configured + has_creds = bool(self.station_id and self.station_password) + if not has_creds: + self.dbus_service['/Status'] = 0 # Idle - not configured + self._update_report_time_ago() + return True + + self.dbus_service['/Status'] = 1 # Active + + # Station location update + if self.api_key and self._should_update_station(lat, lon): + self.logger.info("Updating station location...") + self._update_station(lat, lon) + + # Observation upload + interval = max(300, self.upload_interval) + if now - self.last_observation_time >= interval: + self.dbus_service['/Status'] = 2 # Uploading + if self._send_observation(): + self.last_observation_success = int(now) + self.dbus_service['/LastReportTime'] = ( + self.last_observation_success) + if self.last_observation_time <= now: + self.last_observation_time = now + self.dbus_service['/Status'] = 1 + + self._update_report_time_ago() + + except dbus.exceptions.DBusException as e: + self.logger.warning(f"D-Bus error: {e}") + except Exception as e: + self.logger.exception(f"Unexpected error: {e}") + + return True + + +def main(): + DBusGMainLoop(set_as_default=True) + + print("=" * 60) + print(f"Windy Station v{VERSION}") + print("=" * 60) + + mainloop = None + + def signal_handler(signum, frame): + try: + sig_name = signal.Signals(signum).name + except ValueError: + sig_name = str(signum) + logging.info(f"Received {sig_name}, shutting down...") + if mainloop is not None: + mainloop.quit() + + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + + try: + controller = WindyStationController() + mainloop = GLib.MainLoop() + mainloop.run() + except KeyboardInterrupt: + print("\nShutdown requested") + except Exception as e: + logging.error(f"Fatal error: {e}", exc_info=True) + sys.exit(1) + finally: + logging.info("Service stopped") + + +if __name__ == '__main__': + main() diff --git a/mfd-custom-app/README.md b/mfd-custom-app/README.md new file mode 100644 index 0000000..e399176 --- /dev/null +++ b/mfd-custom-app/README.md @@ -0,0 +1,298 @@ +# Custom MFD App for Venus OS + +Custom pages for the Victron HTML5 MFD app, adding weather, tides, tracking, +and generator dashboards. Designed for the Raymarine Axiom but works with any +MFD that loads the Victron HTML5 app. + +## Custom Pages + +- **Mooring View** - Combined dashboard with weather, tides, generator, and 7-day forecast +- **Weather Station** - Wind speed/gust/direction, temperature, pressure (from Windy Station) +- **Tides** - Tide predictions, charts, and current depth tracking (from dbus-tides) +- **Tide Analysis** - Detailed tide analysis with historical data +- **NFL Tracking** - GPS position, track points, reporting status (from No Foreign Land) +- **Generator Ramp** - Ramp state, current limit, power monitoring, overload detection + +The Mooring View includes a **Forecast Strip** with 7-day Meteoblue weather +forecasts, day/night overlays, wind arrows, and moon phase visualizations. + +Access via the Settings menu (gear icon) > Custom Pages section. + +## Target Device + +- Raymarine Axiom (1280x800, Chrome 106) +- Works at both 640x800 (half-screen) and 1280x800 (full-screen) +- Compatible with light, dark, and night mode themes + +--- + +## Prerequisites + +- Node.js 20+ and npm installed on your build machine +- SSH access to the Cerbo GX (root@cerbo) +- Docker (optional, for containerized builds) + +--- + +## Building Locally (Linux) + +### Install Dependencies + +From the `venus-html5-app/` directory: + +```bash +cd venus-html5-app +CYPRESS_INSTALL_BINARY=0 npm install +``` + +Setting `CYPRESS_INSTALL_BINARY=0` skips the large Cypress binary download +(not needed for builds or dev server). + +### Build for Production + +```bash +CI=false npm run build +``` + +The built app is output to `venus-html5-app/dist/` (static HTML/JS/CSS files). + +### Full Package (tarball + USB zip) + +From the `mfd-custom-app/` directory: + +```bash +./package.sh +``` + +This runs the local build and creates: +- `mfd-custom-app.tar.gz` - For SSH deployment +- `venus-data.zip` - For USB stick deployment + +### Building with Docker (alternative) + +If you prefer a containerized build: + +```bash +cd venus-html5-app +docker build -f Dockerfile.build -t mfd-custom-app-builder . +``` + +Extract the build output: + +```bash +docker create --name mfd-extract mfd-custom-app-builder +docker cp mfd-extract:/app/dist ./dist +docker rm mfd-extract +``` + +--- + +## Development Server (Live Reload) + +The webpack dev server recompiles and refreshes the browser automatically +when you edit source files. This is the recommended workflow for active +development. + +### Start the dev server + +From the `venus-html5-app/` directory: + +```bash +npm start +``` + +This starts a webpack-dev-server on `http://0.0.0.0:3000/app/` with +hot module replacement (HMR). Changes to `.tsx`, `.ts`, `.css`, and `.scss` +files are reflected in the browser instantly without a full page reload. + +To test with live Cerbo data, add your Cerbo IP as a query parameter: + +``` +http://localhost:3000/app/?host=CERBO_IP&port=9001 +``` + +Note: Your Cerbo must have MQTT on LAN enabled (Settings > Services > MQTT on LAN) +and port 9001 (WebSocket) must be reachable from your browser. + +### Stop the dev server + +Press `Ctrl+C` in the terminal where it's running. + +--- + +## Serving a Production Build Locally + +You can serve the built app locally to verify the production build before +deploying to the Cerbo. + +### Start the test server + +From the `venus-html5-app/` directory (after building): + +```bash +npm run serve +``` + +This serves the `dist/` directory on `http://0.0.0.0:3001/app/`. + +To use Docker instead: + +```bash +docker-compose -f docker-compose.test.yaml up -d +``` + +### Open in browser + +``` +http://localhost:3001/app/ +``` + +The app will load but won't connect to MQTT (no Cerbo available). +To test with live data, pass your Cerbo IP as a query parameter: + +``` +http://localhost:3001/app/?host=CERBO_IP&port=9001 +``` + +### Stop the test server + +For the Node server, press `Ctrl+C` in the terminal. For Docker: + +```bash +docker-compose -f docker-compose.test.yaml down +``` + +--- + +## Installation on Cerbo GX + +The custom app installs to `/data/www/app/` which overrides the stock Victron app. +The stock app at `/var/www/venus/app/` is never modified and remains available. + +### Option 1: SSH Deployment + +**Step 1 - Copy the tarball to the Cerbo:** + +```bash +scp mfd-custom-app.tar.gz root@cerbo:/data/ +``` + +**Step 2 - Extract and install:** + +```bash +ssh root@cerbo +cd /data +tar xzf mfd-custom-app.tar.gz +cd mfd-custom-app +./install.sh +``` + +The install script will: +1. Back up any existing custom app at `/data/www/app/` +2. Copy the built app files to `/data/www/app/` +3. Add an entry to `/data/rc.local` so the app is restored after firmware updates +4. Copy the full package to `/data/mfd-custom-app/` for persistence + +**Step 3 - Verify:** + +Open your MFD or navigate to `http://CERBO_IP/app/` in a browser. +The custom pages should be accessible from the Settings menu (gear icon). + +### Option 2: Quick Deploy (no tarball) + +If you already extracted the build output to `mfd-custom-app/app/`: + +```bash +scp -r mfd-custom-app root@cerbo:/data/ +ssh root@cerbo "cd /data/mfd-custom-app && ./install.sh" +``` + +### Option 3: USB Stick + +1. Copy `venus-data.zip` to a FAT32-formatted USB drive +2. Insert the USB drive into the Cerbo GX +3. Reboot the Cerbo +4. The Cerbo auto-extracts the zip contents to `/data/` + +--- + +## Uninstallation + +```bash +ssh root@cerbo +cd /data/mfd-custom-app +./uninstall.sh +``` + +The uninstall script will: +1. Remove `/data/www/app/` (the custom override) +2. Restore any previously backed-up custom app +3. Remove the rc.local entry +4. Remove `/data/mfd-custom-app/` + +The stock Victron app is immediately restored at `http://CERBO_IP/app/`. + +### Quick Restore (without uninstall script) + +If something goes wrong and you just need the stock app back: + +```bash +ssh root@cerbo "rm -rf /data/www/app" +``` + +That's all it takes. Removing the override directory restores the stock app instantly. +No reboot needed - just refresh the MFD page. + +--- + +## File Layout + +``` +venus-html5-app/ # Forked Victron HTML5 app (source) + Dockerfile.build # Docker build environment (Node 20) + docker-compose.test.yaml # Local test server + src/app/Marine2/ + modules/AppViews/ # Modified: custom view enum values + components/ + ui/SettingsMenu/ # Modified: custom page navigation buttons + views/custom/ # New: all custom view components + CombinedView.tsx # Mooring View (combined dashboard) + WeatherView.tsx # Windy Station weather data + TideView.tsx # Tide predictions and charts + TideAnalysisView.tsx # Detailed tide analysis + NflTrackingView.tsx # No Foreign Land GPS tracking + GeneratorRampView.tsx # Generator ramp control + ForecastStrip.tsx # 7-day forecast timeline with moon phases + CompactTideCard.tsx # Compact tide summary card + WindCompass.tsx # Wind direction compass + tide-helpers.ts # Tide calculation utilities + utils/hooks/ + use-custom-mqtt.ts # New: MQTT hooks for custom D-Bus services + +mfd-custom-app/ # Deployment package + install.sh # Install on Cerbo GX + uninstall.sh # Remove from Cerbo GX + package.sh # Build + package tarball and USB zip + README.md # This file +``` + +## Architecture + +The custom pages integrate into the existing venus-html5-app (Marine2 variant): +- New views added to `AppViews` enum for routing +- MQTT subscriptions to custom D-Bus services via `use-custom-mqtt.ts` +- Navigation via the Settings menu (gear icon) +- Uses existing Tailwind breakpoints and Victron color system +- Responsive: works at 640x800 (half-screen) and 1280x800 (full-screen) + +### MQTT Topics + +The custom views subscribe to these D-Bus services bridged to MQTT: + +| Service | MQTT Prefix | Data | +|---------|-------------|------| +| Windy Station | `N/{id}/windystation/0/` | Wind, temp, pressure | +| Tides | `N/{id}/tides/0/` | Tide predictions, depth, harmonic data | +| Meteoblue Forecast | `N/{id}/meteoblueforecast/0/` | 7-day weather, sun/moon times, moon phases | +| NFL Tracking | `N/{id}/nfltracking/0/` | GPS, track points, status | +| Generator Ramp | `N/{id}/generatorramp/0/` | State, current, power, overloads | diff --git a/mfd-custom-app/apply-nginx-override.sh b/mfd-custom-app/apply-nginx-override.sh new file mode 100644 index 0000000..3f6a87d --- /dev/null +++ b/mfd-custom-app/apply-nginx-override.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# +# Apply nginx location override to serve the custom MFD app from /data/www/app/ +# +# VenusOS generates its nginx config at runtime in /run/nginx/sites-enabled/. +# The stock config sets root to /var/www/venus, so /app/ serves the stock app +# from /var/www/venus/app/. This script injects a location block so /app/ +# serves from /data/www/app/ instead, with the stock app at /default/app/. +# +# Called from rc.local on boot (backgrounded) and directly during install. + +[ -d /data/www/app ] || exit 0 + +# Wait for nginx runtime config to be generated (may take a moment on boot) +n=0 +while [ $n -lt 60 ]; do + conf=$(ls /run/nginx/sites-enabled/* 2>/dev/null | head -n 1) + [ -n "$conf" ] && break + n=$((n + 1)) + sleep 1 +done + +if [ -z "$conf" ]; then + logger -t mfd-custom-app "nginx config not found after 60s, cannot apply override" + exit 1 +fi + +# Skip if already applied +grep -q "mfd-custom-app-override" "$conf" && exit 0 + +# Write location blocks to a temp file (avoids $uri being expanded by the shell) +cat > /tmp/mfd-nginx-override.txt << 'OVERRIDE' + + # mfd-custom-app-override + location ^~ /app/ { root /data/www; } + location ^~ /default/app/ { alias /var/www/venus/app/; } +OVERRIDE + +# Inject after each 'root /var/www/venus;' (applies to both http and https blocks) +sed -i '/root \/var\/www\/venus;/r /tmp/mfd-nginx-override.txt' "$conf" +rm -f /tmp/mfd-nginx-override.txt + +nginx -s reload 2>/dev/null +logger -t mfd-custom-app "nginx override applied, custom app serving at /app/" diff --git a/mfd-custom-app/install.sh b/mfd-custom-app/install.sh new file mode 100755 index 0000000..8413fa9 --- /dev/null +++ b/mfd-custom-app/install.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# +# Install Custom MFD App for Venus OS +# +# Deploys the custom HTML5 app to /data/www/app/ which overrides +# the default Victron app. The original remains at /default/app/. +# +# Usage: +# ./install.sh + +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +APP_SOURCE="$SCRIPT_DIR/app" +INSTALL_DIR="/data/www/app" +BACKUP_DIR="/data/www/app.backup" +RC_LOCAL="/data/rc.local" + +echo "==================================================" +echo "Custom MFD App - Installation" +echo "==================================================" + +if [ ! -d "/data" ]; then + echo "ERROR: /data not found. Not a Venus OS device." + exit 1 +fi + +if [ ! -d "$APP_SOURCE" ]; then + echo "ERROR: App files not found in $APP_SOURCE" + echo " Run package.sh first to build the app." + exit 1 +fi + +echo "1. Backing up existing custom app (if any)..." +if [ -d "$INSTALL_DIR" ]; then + if [ -d "$BACKUP_DIR" ]; then + rm -rf "$BACKUP_DIR" + fi + cp -r "$INSTALL_DIR" "$BACKUP_DIR" + echo " Backed up to $BACKUP_DIR" +else + echo " No existing custom app found" +fi + +echo "2. Installing custom MFD app..." +mkdir -p "$INSTALL_DIR" +rm -rf "${INSTALL_DIR:?}/"* +cp -r "$APP_SOURCE/"* "$INSTALL_DIR/" +echo " Installed to $INSTALL_DIR" + +echo "3. Copying package to /data for persistence..." +if [ "$SCRIPT_DIR" != "/data/mfd-custom-app" ]; then + mkdir -p /data/mfd-custom-app + cp -r "$SCRIPT_DIR/"* /data/mfd-custom-app/ + echo " Copied to /data/mfd-custom-app" +else + echo " Already in /data/mfd-custom-app" +fi + +echo "4. Setting up rc.local for persistence..." +if [ ! -f "$RC_LOCAL" ]; then + echo '#!/bin/bash' > "$RC_LOCAL" + chmod +x "$RC_LOCAL" +fi + +# Remove any existing mfd-custom-app entries (clean upgrade) +if grep -q "# BEGIN mfd-custom-app" "$RC_LOCAL"; then + sed -i '/# BEGIN mfd-custom-app/,/# END mfd-custom-app/d' "$RC_LOCAL" +elif grep -q "mfd-custom-app" "$RC_LOCAL"; then + TEMP_RC=$(mktemp) + grep -v "mfd-custom-app" "$RC_LOCAL" | grep -v "# Custom MFD App" > "$TEMP_RC" || true + mv "$TEMP_RC" "$RC_LOCAL" + chmod +x "$RC_LOCAL" +fi + +cat >> "$RC_LOCAL" << 'RCLOCAL' + +# BEGIN mfd-custom-app +if [ -d /data/mfd-custom-app/app ] && [ ! -d /data/www/app ]; then + mkdir -p /data/www/app + cp -r /data/mfd-custom-app/app/* /data/www/app/ +fi +if [ -x /data/mfd-custom-app/apply-nginx-override.sh ]; then + /data/mfd-custom-app/apply-nginx-override.sh & +fi +# END mfd-custom-app +RCLOCAL +echo " Added to rc.local for persistence" + +echo "5. Applying nginx override..." +if [ -x "$SCRIPT_DIR/apply-nginx-override.sh" ]; then + "$SCRIPT_DIR/apply-nginx-override.sh" +elif [ -x /data/mfd-custom-app/apply-nginx-override.sh ]; then + /data/mfd-custom-app/apply-nginx-override.sh +fi +echo " Custom app now serving at /app/" + +echo "" +echo "==================================================" +echo "Installation complete!" +echo "==================================================" +echo "" +echo "Custom app: http://CERBO_IP/app/" +echo "Original app: http://CERBO_IP/default/app/" +echo "" +echo "To uninstall: ./uninstall.sh" +echo "" diff --git a/mfd-custom-app/package.sh b/mfd-custom-app/package.sh new file mode 100755 index 0000000..fdb7a98 --- /dev/null +++ b/mfd-custom-app/package.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# +# Package script for Custom MFD App +# +# Builds the venus-html5-app and packages it into a distributable tarball +# and a venus-data.zip for USB stick deployment. +# +# Usage: +# ./package.sh + +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +HTML5_DIR="$PROJECT_ROOT/venus-html5-app" +DIST_NAME="mfd-custom-app" +OUTPUT_DIR="$PROJECT_ROOT" + +echo "==================================================" +echo "Packaging Custom MFD App" +echo "==================================================" + +if [ ! -d "$HTML5_DIR" ]; then + echo "ERROR: venus-html5-app not found at $HTML5_DIR" + exit 1 +fi + +if [ ! -f "$HTML5_DIR/package.json" ]; then + echo "ERROR: package.json not found in $HTML5_DIR" + exit 1 +fi + +echo "1. Building app locally..." +cd "$HTML5_DIR" +CI=false npm run build 2>&1 + +BUILD_DIR="$HTML5_DIR/dist" +if [ ! -d "$BUILD_DIR" ]; then + echo "ERROR: Build directory not found at $BUILD_DIR" + echo " The build may have failed." + exit 1 +fi + +echo "3. Assembling package..." +TEMP_DIR=$(mktemp -d) +PKG_DIR="$TEMP_DIR/$DIST_NAME" +mkdir -p "$PKG_DIR/app" + +cp -r "$BUILD_DIR/"* "$PKG_DIR/app/" +cp "$SCRIPT_DIR/install.sh" "$PKG_DIR/" +cp "$SCRIPT_DIR/uninstall.sh" "$PKG_DIR/" +cp "$SCRIPT_DIR/apply-nginx-override.sh" "$PKG_DIR/" +chmod +x "$PKG_DIR/install.sh" "$PKG_DIR/uninstall.sh" "$PKG_DIR/apply-nginx-override.sh" +if [ -f "$SCRIPT_DIR/README.md" ]; then + cp "$SCRIPT_DIR/README.md" "$PKG_DIR/" +fi + +echo "4. Creating tarball..." +cd "$TEMP_DIR" +tar -czf "$OUTPUT_DIR/${DIST_NAME}.tar.gz" "$DIST_NAME" +echo " Created: $OUTPUT_DIR/${DIST_NAME}.tar.gz" + +echo "5. Creating venus-data.zip for USB deployment..." +USB_DIR="$TEMP_DIR/venus-data" +mkdir -p "$USB_DIR/www/app" +cp -r "$BUILD_DIR/"* "$USB_DIR/www/app/" +cd "$TEMP_DIR" +zip -r "$OUTPUT_DIR/venus-data.zip" "venus-data" -x "*.DS_Store" +echo " Created: $OUTPUT_DIR/venus-data.zip" + +rm -rf "$TEMP_DIR" + +echo "" +echo "==================================================" +echo "Package created!" +echo "==================================================" +echo "" +echo "Deploy via SSH:" +echo " scp $OUTPUT_DIR/${DIST_NAME}.tar.gz root@cerbo:/data/" +echo " ssh root@cerbo 'cd /data && tar xzf ${DIST_NAME}.tar.gz && cd ${DIST_NAME} && ./install.sh'" +echo "" +echo "Deploy via USB:" +echo " Copy venus-data.zip to a USB stick, insert into Cerbo, reboot." +echo "" diff --git a/mfd-custom-app/uninstall.sh b/mfd-custom-app/uninstall.sh new file mode 100755 index 0000000..2f7be1b --- /dev/null +++ b/mfd-custom-app/uninstall.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# +# Uninstall Custom MFD App for Venus OS +# +# Removes the custom HTML5 app override, restoring the default Victron app. +# +# Usage: +# ./uninstall.sh + +set -e + +INSTALL_DIR="/data/www/app" +BACKUP_DIR="/data/www/app.backup" +PERSIST_DIR="/data/mfd-custom-app" +RC_LOCAL="/data/rc.local" + +echo "==================================================" +echo "Custom MFD App - Uninstall" +echo "==================================================" + +echo "1. Removing custom app..." +if [ -d "$INSTALL_DIR" ]; then + rm -rf "$INSTALL_DIR" + echo " Removed $INSTALL_DIR" +else + echo " Not installed" +fi + +echo "2. Removing nginx override..." +NGINX_CONF=$(ls /run/nginx/sites-enabled/* 2>/dev/null | head -n 1) +if [ -n "$NGINX_CONF" ] && grep -q "mfd-custom-app-override" "$NGINX_CONF"; then + sed -i '/mfd-custom-app-override/d' "$NGINX_CONF" + sed -i '/location \^~ \/app\/ { root \/data\/www/d' "$NGINX_CONF" + sed -i '/location \^~ \/default\/app\//d' "$NGINX_CONF" + nginx -s reload 2>/dev/null || true + echo " Nginx override removed" +else + echo " No nginx override found" +fi + +echo "3. Restoring backup (if available)..." +if [ -d "$BACKUP_DIR" ]; then + mv "$BACKUP_DIR" "$INSTALL_DIR" + echo " Restored from backup" +else + echo " No backup to restore. Default Victron app will be used." +fi + +echo "4. Cleaning up rc.local..." +if [ -f "$RC_LOCAL" ]; then + if grep -q "# BEGIN mfd-custom-app" "$RC_LOCAL"; then + sed -i '/# BEGIN mfd-custom-app/,/# END mfd-custom-app/d' "$RC_LOCAL" + else + TEMP_RC=$(mktemp) + grep -v "mfd-custom-app" "$RC_LOCAL" | grep -v "# Custom MFD App" > "$TEMP_RC" || true + mv "$TEMP_RC" "$RC_LOCAL" + fi + chmod +x "$RC_LOCAL" + echo " Cleaned rc.local" +else + echo " No rc.local found" +fi + +echo "5. Removing persistent copy..." +if [ -d "$PERSIST_DIR" ]; then + rm -rf "$PERSIST_DIR" + echo " Removed $PERSIST_DIR" +else + echo " No persistent copy found" +fi + +echo "" +echo "==================================================" +echo "Uninstall complete!" +echo "==================================================" +echo "" +echo "The default Victron app is restored at: http://CERBO_IP/app/" +echo "" diff --git a/venus-html5-app/.env b/venus-html5-app/.env new file mode 100644 index 0000000..5c90023 --- /dev/null +++ b/venus-html5-app/.env @@ -0,0 +1,10 @@ +PORT=3000 +PUBLIC_URL=/app +BUILD_PATH=./dist +# Whitelabels: +# 1. "KVNRV" - Generic RV-themed whitelabel +# 2. "Marine" - the original theme of the Venus HTML5 App +# 3. "Marine2" - the new Venus HTML5 App theme +REACT_APP_WHITELABEL=Marine2 +# Allow overriding the language via the ?lang= URL parameter (disabled in production) +REACT_APP_ENABLE_LANG_OVERRIDE=true diff --git a/venus-html5-app/.env.local.example b/venus-html5-app/.env.local.example new file mode 100644 index 0000000..0a4ebe8 --- /dev/null +++ b/venus-html5-app/.env.local.example @@ -0,0 +1,12 @@ +PORT=8000 +PUBLIC_URL=/app +# Whitelabels: +# 1. "KVNRV" - Generic RV-themed whitelabel +# 2. "Marine" - the original theme of the Venus HTML5 App +# 3. "Marine2" - the new Venus HTML5 App theme +REACT_APP_WHITELABEL=Marine2 +POEDITOR_API_URL=https://api.poeditor.com/v2/ +POEDITOR_PROJECT_ID=457055 +POEDITOR_API_KEY= +# Allow overriding the language via the ?lang= URL parameter (disabled in production) +REACT_APP_ENABLE_LANG_OVERRIDE=true diff --git a/venus-html5-app/.github/workflows/ci.yml b/venus-html5-app/.github/workflows/ci.yml new file mode 100644 index 0000000..2f12741 --- /dev/null +++ b/venus-html5-app/.github/workflows/ci.yml @@ -0,0 +1,95 @@ +name: Continous Integration Workflow +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Node.js 20.x + uses: actions/setup-node@master + with: + node-version: 20.x + - name: Node modules cache + uses: actions/cache@v4 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + - name: Install the dependencies + run: npm ci + - name: Build the bundle for testing + run: npm run build + env: + PUBLIC_URL: / + REACT_APP_ENABLE_LANG_OVERRIDE: false + - name: Serve the application + run: npm run serve & + - name: Run the Venus simulation + run: > + docker run -d --rm -p 9001:9001 -p 1883:1883 -p 3000:3000 -p 8080:80 + --name venus-docker victronenergy/venus-docker:latest + /root/run_with_simulation.sh z + - name: Run unit and E2E tests + run: npm run test:ci + - name: Generate E2E Report + run: npm run generate-e2e-report + if: always() + - name: Upload E2E Results + uses: actions/upload-artifact@v4 + if: always() + with: + name: e2e-tests-results + path: cypress/results/results.json + - name: Upload E2E Report + uses: actions/upload-artifact@v4 + if: always() + with: + name: e2e-tests-report + path: cypress/report + - name: Upload Cypress Screenshots + uses: actions/upload-artifact@v4 + if: always() + with: + name: cypress-screenshots + path: cypress/screenshots + - name: Upload Cypress Videos + uses: actions/upload-artifact@v4 + if: always() + with: + name: cypress-videos + path: cypress/videos + - name: Check Tag + id: check_tag + run: | + if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo ::set-output name=match::true + fi + - name: Build the production bundle + if: steps.check_tag.outputs.match == 'true' + run: npm run build + env: + PUBLIC_URL: /app + REACT_APP_ENABLE_LANG_OVERRIDE: false + - name: Archive bundle + if: steps.check_tag.outputs.match == 'true' + run: tar -zcvf venus-html5-app-${{ github.ref_name }}.tar.gz dist/ + - name: Create Release + id: create_release + if: steps.check_tag.outputs.match == 'true' + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + draft: true + - name: Upload Release Archive + if: steps.check_tag.outputs.match == 'true' + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./venus-html5-app-${{ github.ref_name }}.tar.gz + asset_name: venus-html5-app-${{ github.ref_name }}.tar.gz + asset_content_type: application/zip diff --git a/venus-html5-app/.gitignore b/venus-html5-app/.gitignore new file mode 100644 index 0000000..d0cb486 --- /dev/null +++ b/venus-html5-app/.gitignore @@ -0,0 +1,38 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/**/node_modules +/.pnp + +# testing +/coverage + +# production +/build +/dist +/www +venus-data.zip +venus-html5-app.tar.gz + +# misc +.DS_Store +.env.local +.env.development.local +.env.test.local +.env.production.local + +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +cypress/results +cypress/screenshots +cypress/videos +cypress/report + +.idea +.husky +.vscode + +# deploy scripts +networks.csv diff --git a/venus-html5-app/.npmrc b/venus-html5-app/.npmrc new file mode 100644 index 0000000..521a9f7 --- /dev/null +++ b/venus-html5-app/.npmrc @@ -0,0 +1 @@ +legacy-peer-deps=true diff --git a/venus-html5-app/.prettierrc b/venus-html5-app/.prettierrc new file mode 100644 index 0000000..afa7d06 --- /dev/null +++ b/venus-html5-app/.prettierrc @@ -0,0 +1,4 @@ +{ + "semi": false, + "printWidth": 120 +} diff --git a/venus-html5-app/.prettierrc.json b/venus-html5-app/.prettierrc.json new file mode 100644 index 0000000..bc5fbce --- /dev/null +++ b/venus-html5-app/.prettierrc.json @@ -0,0 +1,6 @@ +{ + "semi": false, + "singleQuote": false, + "printWidth": 120, + "trailingComma": "es5" +} diff --git a/venus-html5-app/CONTRIBUTING.md b/venus-html5-app/CONTRIBUTING.md new file mode 100644 index 0000000..b23e8dc --- /dev/null +++ b/venus-html5-app/CONTRIBUTING.md @@ -0,0 +1,57 @@ +## Proper commit messages are important + +Seven rules of making a great commit message: + +1. Limit the subject line to around 60 characters +2. Separate subject from body with a blank line +3. Do not end the subject line with a period +4. Use the imperative mood in the subject line +5. Wrap the body at 80 characters +6. Use the body to explain what and why vs. how +7. Commit one change at a time. For example changing a default setting, adding something new and fixing a bug are three separate commits. + +More info here in a post from [Chris Beam](https://chris.beams.io/posts/git-commit/) and there is, obviously, much more to find about this all over the internet. + +## Branches and workflow + +Work in progress should be done in a branch, prefix it with WIP, or work in your own forked repo. +We have a rebase-style workflow. Not a merge one. +Most important: once ready for checking / discussing or inclusion into the master branch: first squash / rebase / split / clean / reword. See cleaning up section below. + +Releases are in the master branch +Don't push -f in a master branch (duh.. ). But don't by shy on push -f in your work branches either. +Once code is included in the master branch, remove the working branch + +## Cleaning up before review and/or inclusions in master + +Often, when working on a feature for a while, the git history will look similar to this (oldest on top): + +1. do a +2. do b and c +3. readme: add explanation +4. fix a +5. readme: improve explanation +6. etcetera + +Before asking anyone to review, and also before putting that in master, it needs to be cleaned up. Meaning some commits can be combined, others perhaps split up, and also check all the commit titles and messages. See above seven rules for guidelines on that. + +Back to the example: there is no point in cluttering up the history of that project with item 4 and 5 in above list. 4 needs to be squashed with 1, and perhaps number 2 needs to split up. To do this, use git rebase -i, and afterwards push -f to push your WIP branch to the remote. + +In case you want to preserve history while working still working on the issue, then simply number the branches. Ie start a new one every time: WIP-bbb1, WIP-bbb2 etcetera. Same when working on a feature for a long time, on an active project: it might make sense to now and then rebase your work on master to make sure there are no conflicts. Steps to take in that case are: + +1. Update your master branch (git fetch origin master:master, or more simples, git checkout master && git pull && git checkout your-wip-branch) +2. (optionally) rename your branch: git checkout -b WIP-mywork-2 +3. Put your commits on top of the updated master branch: git rebase master + +## Releasing and tagging + +1. Tag it: git tag -a 1.20 + +* do not put an underscore in the version number, fe 1_20 is not good +* make sure to stick to whatever the default tags names in your project are + +2. After hitting enter, the editor for the tag message shows up, in there, put the change log. +3. Push the tag: git push origin 1.20 +4. Add the same change log to the [wiki](https://github.com/victronenergy/venus-private/wiki/todo) + +More info can be found here: https://github.com/victronenergy/venus/wiki/release-cycle \ No newline at end of file diff --git a/venus-html5-app/Dockerfile.build b/venus-html5-app/Dockerfile.build new file mode 100644 index 0000000..d569e57 --- /dev/null +++ b/venus-html5-app/Dockerfile.build @@ -0,0 +1,19 @@ +FROM node:20-slim + +WORKDIR /app + +COPY package.json package-lock.json ./ + +ENV CYPRESS_INSTALL_BINARY=0 + +RUN npm install -g npm@11.10.1 +RUN npm ci --ignore-scripts 2>&1 || npm install --ignore-scripts 2>&1 + +COPY . . + +ENV NODE_ENV=production +ENV GENERATE_SOURCEMAP=false + +RUN node scripts/build.js + +CMD ["echo", "Build complete"] diff --git a/venus-html5-app/LICENSE b/venus-html5-app/LICENSE new file mode 100644 index 0000000..d15a8b5 --- /dev/null +++ b/venus-html5-app/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Victron Energy BV + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/venus-html5-app/README.md b/venus-html5-app/README.md new file mode 100644 index 0000000..23f63e4 --- /dev/null +++ b/venus-html5-app/README.md @@ -0,0 +1,247 @@ +![screenshot](/legacy/victron-webapp-screenshot.png?raw=true) + +# Venus OS hosted web app + +The "app" is a single page application that communicates to the rest of Venus OS via MQTT over websockets. +It uses React with ES6 for the UI layer and also includes a wrapper service for the MQTT interface. + +Its primary purpose is to be a simple to use and nice looking UI for a Victron system on +marine Multi Functional Displays, such as the ones from Garmin, Simrad and others. This +removes the need for a Victron panel at the helm: less clutter on the dashboard. + +The secondary purpose is to help OEMs, boat builders and motorhome builders for example, +make their own custom UI. + +## 0. Contents + +Chapters in this readme: + +1. Functionality. +2. Development +3. Testing +4. Making a release +5. Device error logging +6. Device debugging + +## 1. Functionality + +### 1.1 Documentation per Box + +`Boxes` are the various designed visualisation: there are a Battery box, a Generator box, a Tanks box, and so forth. + +How certain devices are visualised/rendered on HTML5, ie. in to what box(es), and what topics are used for what parameter, and more is documented in three places: + +- [TOPICS.md](https://github.com/victronenergy/venus-html5-app/blob/master/TOPICS.md) +- [wiki/Translating system components into the HTML5 app](https://github.com/victronenergy/venus-html5-app/wiki/Translating-system-components-into-the-HTML5-app) +- [wiki/Dashboard overview](https://github.com/victronenergy/venus-html5-app/wiki/Dashboard-overview) + +### 1.2 Handling disconnects + +When devices are disconnected from the GX Device, see [this issue](https://github.com/victronenergy/venus-html5-app/issues/49) +for what happens on the D-Bus. + +On MQTT, this is translated into sending an empty message on the subs which depend on the lost service. Depending on the element we show either -- for the value (in the case of numeric values) or another default state for the component (like disconnected for the active source). + +## 2. Development + +### 2.1 Initial setup + +If it's the first time you run the app: + +- make sure to have `node` & `npm` installed on your machine +- run `npm install` in the root folder + +### 2.2 Setting white label + +This repository contains several white-label apps. To work with a specific app, set the correct `APP` environment variable in the `.env.local` file. For example, to build the `Marine2` app, use: + +``` +REACT_APP_WHITELABEL=Marine2 +``` + +You can find the list of available apps in the `.env.local.example` file. + +### 2.3 Run app locally + +To run the app locally for development, run: + +`npm run start` + +And then open the app in the browser at `http://localhost:8000`. + +This will start the webpack dev server, which will recompile the app on code changes and hot reload the UI. + +Note that the app will attempt to connect to MQTT broker served via the same port as the app and path `/websocket-mqtt`, and that will eventually fail. + +You will need to change the `host`, `port`, and `path` (defaults to `/websocket-mqtt`) query parameters to connect to a different Venus websocket MQTT host. + +To connect to a Venus device with `VENUS_DEVICE_IP` running firmware >= 3.50 use the following URL: + +`http://localhost:8000?host=&port=80` + +To connect to Venus device with `VENUS_DEVICE_IP` running firmware < 3.50, or to a `venus-docker` simulation, use the following URL: + +`http://localhost:8000/app?host=VENUS_DEVICE_IP&port=9001&path=%02%03` + +Note: the `port` needs to be overriden to connect directly to `flashmq` provided websocket port. + +Note: the `path` URL parameter requires special sequence `%02%03` in order to properly override the default `/websocket-mqtt` path and specify an empty string. + +This way you can run the local app against any Venus device that is reachable via your network. + +### 2.4 Using Demo mode on Venus device + +Every Venus device also has a Demo mode, which allows you to get useful data if you only have the Venus device available, without requiring various Victron devices to be connected to the Venus device. To enable it, navigate to the `Venus Remote Console` -> `Settings` -> `General`. + +### 2.5 Running the app with no Venus device available + +Use [venus-docker](https://github.com/victronenergy/venus-docker) in demo mode. + +You can run multiple `venus-docker` simulations by executing: `echo {a..z} | xargs -n1 ./run.sh -s`. Each container running a simulation will expose MQTT on +an increasing port number starting from `9001`. + +### 2.6 Metrics available + +- Identify the D-bus channel that you want to read [from here](https://github.com/victronenergy/venus/wiki/dbus) +- Create a component using MqttSubscriptions or MqttTopicWildcard and pass the topic as the wrapper topic. See examples in other components + +### 2.7 Deploying to a device during development + +#### 2.7.1 Get the device ip + +In order to deploy you need to know the target device's IP. It can be connected to by ethernet, LAN or the device's own WLAN. +Instructions on how to find the IPs can be found [here](https://www.victronenergy.com/media/pg/Venus_GX/en/accessing-the-gx-device.html) +for the Venus GX device. + +The default device's IP address is `172.24.24.1` + +#### 2.7.2 Run deploy script + +In the project main folder run `./bin/deploy.sh --build ` where ip is the target device's IP. The script also accepts an additional +`--user|-u` param that defines the user for the deployment connection. This defaults to `root`. You will also need a password to connect to the device. To set the password, navigate to the `Venus Remote Console` -> `Settings` -> `General -> Set root password`. + +The deploy script also bundles the app if `--build` or `-b` . Note that the script assumes that it's run from the root folder of the application. + +#### 2.7.3 Deploying on multiple devices + +To deploy the app on multiple devices, use `./bin/deploy-multiple.sh` script. This script uses a list of WiFi access points defined in `network.csv` file. + +#### 2.7.4 Deploying using a USB stick + +Since Venus OS 2.80, placing the build of the app in `/data/www/app` allows for serving a different version of the app than the one bundled with Venus OS at `/var/www/venus/app`. When the `/data/www/app` is present, it'll be server at `venus.local/app` and the original application at `venus.local/default/app`. + +By creating an archive named `venus-data.zip` that contains the build files from the `dist` inside an `www/app/` folder will ensure that the `/data/www/app` folder will be created and the content of the archived extracted when the GX device is rebooted. + +The content of the `/data` partition is persistent across firmware updates. + +To create the archive, run `./bin/pack.sh` from the root folder of the application. This will create a `venus-data.zip` file. Place this file on a USB stick and insert it into the GX device, then reboot the device. + +### 2.8 Translations + +#### 2.8.1 Syncronizing the translations files with the POEditor Project + +[POEditor](https://poeditor.com/) is used as localization management platform for this project. In order to sync the translations using the scripts from the `poeditor` folder, an API key has to be placed in the `.env.local` according to the `.env.local.example` file. + +#### 2.8.2 Pushing the local translation files to POEditor + +``` +npm run poeditor:push +``` + +Running the command will trigger the following actions: + +1. Add the terms of the main language file (default: en) +1. Add new languges to the POEditor project if they are available locally but missing in POEditor +1. Add the local translations for all the languages +1. Mark translations as fuzzy if there are changes in the translation of the main language + +``` +npm run poeditor:push -f +``` + +Running the comamnd with the `-f` flag will delete the terms from POEditor that are not present in the local file. +Please use with caution. If wrong data is sent, existing terms and their translations might be irreversibly lost. + +#### 2.8.3 Pulling the POEditor translations locally + +``` +npm run poeditor:pull +``` + +## 3. Testing + +### 3.1 Venus OS Release test plan + +In the Venus OS release test plan there is a tab containing all tests. + +### 3.2 Enzyme + +Most components have Enzyme unit tests. Run all of these tests with `npm run test:unit` + +### 3.3 Cypress + +Cypress is used to run integration tests on the compiled ui to make sure it opens and operated correctly in different +display sizes. To run cypress you need to run the live server and an instance of venus docker in the Venus GX demo mode (z): + +(in html5 app repo): `npm run start` + +(in venus docker repo): `./run.sh -s z` + +Then you can run the cypress UI interactively with `npm run cy:open`. + +To run the ui tests in CI-style use `npm run test:e2e` + +### 3.4 Simulating/Debugging MFD UI locally + +MFDs ship web browsers based on `AppleWebKit/537` or `AppleWebKit/601`. + +Partially Simulating the MFD web browser can be achieved by running any browser using the same engine. + +Google Chrome 49 runs `AppleWebKit/537`, so contains all the same CSS/JS limitations, and can be use to simulate/debug CSS issues faster. + +## 4. Making a release + +Whenever a new tag is created, GitHub Actions will build the app, archive the built files and upload them as `venus-html5-app-.tar.gz` to the Github Release associated with the tag. +The app can then be downloaded from `https://github.com/victronenergy/venus-html5-app/releases/download//venus-html5-app-.tar.gz`. +The build script expects the tags to follow semantic versioning (e.g. `1.2.3`, `1.2`, etc.) and will not trigger for tags that don't follow this convention (e.g. `v1.0`, `test`). + +To include the HTML5 app in the next Venus OS version: + +1. Increment the version number in `package.json`. Create and push a commit. +2. Create & push a tag for the version (do not use `v` in the version name, just the number). +3. Edit the generated release, add the changelog, change title (see previous releases) and set checkbox "Set as pre-release" to true. +4. Update the [todo](https://github.com/victronenergy/venus-private/wiki/todo) page for the build. + +You should add a note under `Done - waiting for recipe / venus maintainer`, containing the tag name and the changes included: + +```md +html5-app - + +``` + +For example: + +```md +html5-app - 0.2 \ + * Reworked the UI +``` + +If you need any changes to the how the app is included inside Venus, please specify in the TODO file as well what changes need to be made to the recipe. +All Venus recipes are found [here](https://github.com/victronenergy/meta-victronenergy/tree/master/meta-ve-software/recipes-ve). +A sample recipe for the HTML5 app is [here](https://github.com/victronenergy/meta-victronenergy/tree/master/meta-ve-software/recipes-ve) + +## 5. Device error logging + +When the app is hosted from a Venus device, there is no convenient way to see the errors in the js console. +To make troubleshooting easier the app can send the error messages through websocket port 7890 to the device. +To enable this debugging mode, setup https://github.com/vi/websocat on your Venus device, uncomment the debug code in `index.html`, and deploy to the device. + +## 6. Device debugging + +By adding `debug=true` to the query params you can enable some convenience features for debugging on actual devices: + +- "Reload page" button - refreshes the page +- "Browser info" button - links to page containing basic information about the browser in which the app is running +- A debug log element, which redirects all console messages to a visible element in the ui + +To enable this this debugging mode on a MFD device, uncommend the debug code in `index.html`, override the `debug=true` check, and deploy to the device. diff --git a/venus-html5-app/TOPICS.md b/venus-html5-app/TOPICS.md new file mode 100644 index 0000000..7a1ee61 --- /dev/null +++ b/venus-html5-app/TOPICS.md @@ -0,0 +1,315 @@ +### MQTT TOPICS + +The data originates from D-Bus on Venus OS. Detailed spec is +[here](https://github.com/victronenergy/venus/wiki/dbus). There +is a bridge between D-Bus and MQTT, code & documentation +[here](https://github.com/victronenergy/dbus-mqtt). + +An example of available data is: + +``` +Topic: N/e0ff50a097c0/system/0/Dc/Battery/Voltage` +Payload: {"value": 210} + +Explanation: + +e0ff50a097c0 -> VRM Portal ID +system -> Service type +0 -> Device Instance (used when there is more than one of this service type) +/Dc/Battery/Voltage -> D-Bus path. +``` + +On D-Bus, data is sent on change. And dbus-mqtt translates each transmission on D-Bus, called +a PropertiesChanged message, to a mqtt publish(). + +### What topics have what data? + +First, note that most data comes from the system device. See explanation on the dbus wiki page +for more info on that. Instance is always 0. + +And that most of below paths are copied from that same dbus wiki page; and all lengthy explanations +have been stripped out. So, make sure to see the dbus wiki page for details. + +#### Battery data: + +Info about all batteries available in the system is available under: + +``` +N/{portalId}/system/0/Batteries +// -> {"value": [ + { + "soc": 100.0, + "active_battery_service": true, // True for main battery + "temperature": 30.0, + "power": 472.851013184, + "current": 8.30000019073, + "instance": 256, + "state": 1, + "voltage": 56.9799995422, + "id": "com.victronenergy.battery.ttyO0", + "name": "BMV-702" + } + ]} +``` + +#### Solar data: + +``` +N/{portalId}/system/0/Dc/Pv/Power +N/{portalId}/system/0/Dc/Pv/Current +``` + +#### AC Input data (Shore / Genset) for inverter/charger aka vebus + +There are three types of devices that show up on D-Bus as a com.victronenergy.vebus device: + +- Inverters. These have no AC-input, so no AC-Input current limit +- Multis: one AC-input; connected to either shore (adjustable) or a generator (not adjustable) +- Quattros: two AC-inputs; typically one is connected to shore, and the other to a generator + +Which one you have can be determined from this path: + +``` +N/{portalId}/vebus/{vebusInstanceId}/Ac/NumberOfAcInputs +// 0 = Inverter, 1 = system with one input, 2 = system with two inputs +``` + +##### What input is what (shore, or generator) + +The installer configures the AC-input types in the menus: Settings -> System Setup. + +Then they are stored in 'localsettings', available on MQTT as: + +``` +N/{portalId}/settings/0/Settings/SystemSetup/AcInput1 +N/{portalId}/settings/0/Settings/SystemSetup/AcInput2 +// 0: not in use; 1: grid, 2: generator, 3: shore +``` + +##### Which input is currently active + +Figure out which of the AC-inputs is what; so for example ACinput0 == Generator and +ACinput1 == Shore. (based on the settings above) and then look at this path: + +``` +N/{portalId}/vebus/{vebusInstanceId}/Ac/ActiveIn/ActiveInput +// Active input: 0 = ACin-1, 1 = ACin-2, 240 is none (inverting). +``` + +##### Where to get the readings for the AC inputs (grid / shore / generator)? + +Normally we'd say go to the .system device. But the html5 app UI also wants +voltage and current, which are not available on .system device. Therefore, get +them straight from the .vebus service: + +- figure out which of the AC-inputs is what; so for example ACinput0 == Generator and + ACinput1 == Shore. +- find the ve.bus service (see below). + +Then, use these paths: + +``` +// So, if the ActiveInput == Acinput1, then populate the Shore box with the values from these paths: +N/{portalId}/vebus/{vebusInstanceId}/Ac/ActiveIn/L1/I <- Current (Amps) (+ L2, L3 for 3 phase) +N/{portalId}/vebus/{vebusInstanceId}/Ac/ActiveIn/L1/P <- Power (Watts) +N/{portalId}/vebus/{vebusInstanceId}/Ac/ActiveIn/L1/V <- Voltage (Volts) +``` + +The number of phases, for all boxes related to the vebus device can be retrieved here: +``` +N/${portalId}/vebus/${vebusInstanceId}/Ac/NumberOfPhases +``` + +Note: For other UIs, that show only power, and also do not show generator & shore +at the same time, you can take the info from these paths under system/0: + +``` +N/{portalId}/system/0/Ac/Grid/* <- All from the shore. TODO: check if this shouldn't be /Ac/Shore +N/{portalId}/system/0/Ac/Genset/* <- All from the generator. +``` + +#### Rest of inverter/charger data: + +``` +N/{portalId}/system/0/Dc/Vebus/Current <- charge/discharge current between battery + and inverter/charger + +N/{portalId}/system/0/Dc/Vebus/Power <- same, but then the power + +N/{portalId}/system/0/SystemState/State <- Absorption, float, etc. + 0 - "Off" 1 - "Low power" + 2 - "VE.Bus Fault condition" + 3 - "Bulk charging" + 4 - "Absorption charging" + 5 - "Float charging" + 6 - "Storage mode" + 7 - "Equalisation charging" + 8 - "Passthru" + 9 - "Inverting" + 10 - "Assisting" + 256 - "Discharging" + 257 - "Sustain" + +N/{portalId}/system/0/Ac/ActiveIn/Source <- The active AC-In source of the multi. + 0:not available, 1:grid, 2:generator, + 3:shore, 240: inverting/island mode. + +N/{portalId}/system/0/VebusService <- Returns the service name of the vebus service. + Use that to find the control options: +``` + +#### Inverter/charger control: + +##### Shore input limit + +This starts with finding the service name and instance. + +The VE.Bus device instance is available under: +`N/{portalId}/vebus/+/DeviceInstance // { value: 257 }` + +With above example, the instance is 257, making the prefix `vebus/257`. + +Then the topics you are looking for to control input current limit are: + +``` +N/{portalId}/vebus/{vebusInstanceId}/Ac/In/{n}/CurrentLimit <- R/W for input current limit. +N/{portalId}/vebus/{vebusInstanceId}/Ac/In/{n}/CurrentLimit GetMin <- not implemented! +N/{portalId}/vebus/{vebusInstanceId}/Ac/In/{n}/CurrentLimit GetMax <- not implemented in mqtt +N/{portalId}/vebus/{vebusInstanceId}/Ac/In/{n}/CurrentLimitIsAdjustable + +n = the AC-input. 0 = AC-input 1, and 1 = AC-input 2. + +And, don't hardcode that vebus/257 ;-). +``` + +Notes: + +1. when /vebus/257/In/n/CurrentLimit is set to an invalid value, it will automatically + be changed to a valid value. Some examples for setting it to an invalid value are + setting it too low, or setting it to a value that doesn't match with regards to step-sizes. +2. the Getmin is not implemented because its difficult to implement on our side: The + panel frames used to communicate current limits only contain the max value. min is not + included. We can only get the minimum from the active AC input. The minimum is a special + case, it will change depending on the configuration of the inverter/charger, for example + disabling "powerAssist" changes it. +3. Paths for /CurrentLimit described below are only available starting from version 415 of the VE.Bus device. + +##### Make commonly used shore input limits available as a button + +The list of amperages commonly used depends on wether the system is US based or EU based. To +determine that, get the product id from: + +``` +N/{portalId}/vebus/{vebusInstanceId}/ProductId +``` + +Then, mask the Product id with `0xFF00` + +If the result is `0x1900` or `0x2600` it is an EU model (230VAC) +If the result is `0x2000` or `0x2700` it is an US model (120VAC) + +Default values for US/EU are: + +``` +USAmperage = [10, 15, 20, 30, 50, 100] +EUAmperage = [6, 10, 13, 16, 25, 32, 63] +``` + +##### On/Off/Charger-only control paths + +``` +N/{portalId}/vebus/257/Mode <- Position of the switch. + 1=Charger Only;2=Inverter Only;3=On;4=Off +N/{portalId}/vebus/257/ModeIsAdjustable <- 0: only show the mode / 1: keep it adjustable +``` + +#### AC Loads + +Get the available number of phases: + +``` +N/${portalId}/system/0/Ac/Consumption/NumberOfPhases +``` + +Current: + +``` +N/{portalId]/vebus/{vebusInstanceId}/Ac/Out/L1/I +N/{portalId]/vebus/{vebusInstanceId}/Ac/Out/L2/I +N/{portalId]/vebus/{vebusInstanceId}/Ac/Out/L3/I +``` + +Voltage: + +``` +N/{portalId]/vebus/{vebusInstanceId}/Ac/Out/L1/V +N/{portalId]/vebus/{vebusInstanceId}/Ac/Out/L2/V +N/{portalId]/vebus/{vebusInstanceId}/Ac/Out/L3/V +``` + +Power: + +``` +N/{portalId]/vebus/{vebusInstanceId}/Ac/Out/L1/P +N/{portalId]/vebus/{vebusInstanceId}/Ac/Out/L2/P +N/{portalId]/vebus/{vebusInstanceId}/Ac/Out/L3/P +``` + +#### DC Loads + +``` +Voltage: N/{portalId}/system/0/Dc/Battery/Voltage +Current: power divided by voltage +Power: N/{portalId}/system/0/Dc/System/Power +``` + +#### Charger information + +A system has chargers available if there are devices registered at `/charger/`: + +``` +N/+/charger/+/DeviceInstance +``` + +Then, for each charger, the following info is available: + +``` +// Name displayed is "CustomName", if available, otherwise "ProductName" +N/${portalId}/charger/${deviceInstanceId}/ProductName, +N/${portalId}/charger/${deviceInstanceId}/CustomName + +N/${portalId}/charger/${deviceInstanceId}/Ac/In/CurrentLimit +N/${portalId}/charger/${deviceInstanceId}/Mode // 1-ON, 4-OFF +N/${portalId}/charger/${deviceInstanceId}/State // Same values as system/State + +N/${portalId}/charger/${deviceInstanceId}/NrOfOutputs +N/${portalId}/charger/${deviceInstanceId}/Dc/0/Current, +N/${portalId}/charger/${deviceInstanceId}/Dc/1/Current, +N/${portalId}/charger/${deviceInstanceId}/Dc/2/Current +``` + +The following properties can also be changed: + +``` +W/${portalId}/charger/${deviceInstanceId}/Ac/In/CurrentLimit +W/${portalId}/charger/${deviceInstanceId}/Mode` +``` + +#### Inverter information + +Dbus spec for inverters [here](https://github.com/victronenergy/venus/wiki/dbus) + +For every inverter in the system show an element in the ui. Inverters can be found on D-Bus under two paths: + +- com.victronenergy.inverter (<-- see simulation H in [venus-docker repo](https://github.com/victronenergy/venus-docker)) + - `N/+/charger/+/DeviceInstance` +- com.victronenergy.vebus, with /Ac/NumberOfInputs == 0 (<-- see simulation A in [venus-docker repo](https://github.com/victronenergy/venus-docker)) + - `N/+/vebus/+/DeviceInstance` + +The element should contain this information: + +- The switch (on/off/eco); it wil always be available on D-Bus +- The state (On, Off, Eco, alarm; which can be overload or low battery) +- AC output voltage, current. Not power; because its not available for all models. + +See Inverter.js and Inverters.js for implementation. diff --git a/venus-html5-app/bin/deploy-multiple.sh b/venus-html5-app/bin/deploy-multiple.sh new file mode 100755 index 0000000..4d9b670 --- /dev/null +++ b/venus-html5-app/bin/deploy-multiple.sh @@ -0,0 +1,50 @@ +#!/bin/zsh + +BUILD=false +REBOOT=false +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -b|--build) + BUILD=true + shift # past argument + ;; + esac + case $key in + -r|--reboot) + REBOOT=true + shift # past argument + ;; + esac +done + +if $BUILD; then + echo "Building app.." + npm run build +fi + +# Source: https://gitlab.elnino.tech/elnino/snooze/victron-mfd/-/wikis/Home +# Simrad MFD: 172.25.9.234 +# Raymarine MFD: 172.25.9.64 +# Furuno MFD: 172.25.9.35 +# Garmin MFD: 172.25.9.217 +# Garmin 2 MFD: 172.25.9.122 + +# Associative array +declare -A MFDs +MFDs[Simrad]="172.25.9.234" +MFDs[Raymarine]="172.25.9.64" +MFDs[Furuno]="172.25.9.35" +MFDs[Garmin]="172.25.9.217" +MFDs[Garmin2]="172.25.9.122" + +# Call deploy.sh for each MFD in a loop +for MFD in "${MFDs[@]}"; do + echo "Deploying to $MFD..." + bin/deploy.sh ${MFD} + # Reboot device if requested + if $REBOOT; then + echo "Rebooting $MFD..." + ssh root@${MFD} "reboot" + fi +done \ No newline at end of file diff --git a/venus-html5-app/bin/deploy.sh b/venus-html5-app/bin/deploy.sh new file mode 100755 index 0000000..5ce8a3a --- /dev/null +++ b/venus-html5-app/bin/deploy.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +if [ -z "$1" ]; then + echo "Host not set. Usage: $0 " + exit 1 +fi + +BUILD=false +PORT=22 +POSITIONAL=() +while [[ $# -gt 0 ]]; do + key="$1" + + case $key in + -u|--username) + USERNAME="$2" + shift # past argument + shift # past value + ;; + -b|--build) + BUILD=true + shift # past argument + ;; + -p|--port) + PORT="$2" + shift # past argument + shift # past value + ;; + *) # unknown option + POSITIONAL+=("$1") # save it in an array for later + shift # past argument + ;; + esac +done +set -- "${POSITIONAL[@]}" # restore positional parameters + +USERNAME=${USERNAME:-root} +HOST="$1" + +if $BUILD; then + echo "Building app.." + npm run build +fi + +echo "Uploading dist/* to ${USERNAME}@${HOST}:/data/www/app/" + +echo "mkdir -p /data/www/app/" +ssh -p "${PORT}" -oStrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "${USERNAME}"@"${HOST}" \ + "[ -d /data/www/app/ ] || mkdir -p /data/www/app/" + +rsync --delete --info=progress2 -e "ssh -p ${PORT} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -r dist/* "${USERNAME}"@"${HOST}":/data/www/app/ diff --git a/venus-html5-app/bin/pack.sh b/venus-html5-app/bin/pack.sh new file mode 100755 index 0000000..2262077 --- /dev/null +++ b/venus-html5-app/bin/pack.sh @@ -0,0 +1,8 @@ +npm run build +rm -r www +mkdir www +mkdir www/app +cp -r dist/* www/app +rm venus-data.zip +zip -r venus-data.zip www +rm -r www diff --git a/venus-html5-app/config/env.js b/venus-html5-app/config/env.js new file mode 100644 index 0000000..fae806a --- /dev/null +++ b/venus-html5-app/config/env.js @@ -0,0 +1,102 @@ +const fs = require("fs") +const path = require("path") +const paths = require("./paths") + +// Make sure that including paths.js after env.js will read .env variables. +delete require.cache[require.resolve("./paths")] + +const NODE_ENV = process.env.NODE_ENV +if (!NODE_ENV) { + throw new Error("The NODE_ENV environment variable is required but was not specified.") +} + +// https://github.com/bkeepers/dotenv#what-other-env-files-can-i-use +const dotenvFiles = [ + `${paths.dotenv}.${NODE_ENV}.local`, + // Don't include `.env.local` for `test` environment + // since normally you expect tests to produce the same + // results for everyone + NODE_ENV !== "test" && `${paths.dotenv}.local`, + `${paths.dotenv}.${NODE_ENV}`, + paths.dotenv, +].filter(Boolean) + +// Load environment variables from .env* files. Suppress warnings using silent +// if this file is missing. dotenv will never modify any environment variables +// that have already been set. Variable expansion is supported in .env files. +// https://github.com/motdotla/dotenv +// https://github.com/motdotla/dotenv-expand +dotenvFiles.forEach((dotenvFile) => { + if (fs.existsSync(dotenvFile)) { + require("dotenv-expand")( + require("dotenv").config({ + path: dotenvFile, + }) + ) + } +}) + +// We support resolving modules according to `NODE_PATH`. +// This lets you use absolute paths in imports inside large monorepos: +// https://github.com/facebook/create-react-app/issues/253. +// It works similar to `NODE_PATH` in Node itself: +// https://nodejs.org/api/modules.html#modules_loading_from_the_global_folders +// Note that unlike in Node, only *relative* paths from `NODE_PATH` are honored. +// Otherwise, we risk importing Node.js core modules into an app instead of webpack shims. +// https://github.com/facebook/create-react-app/issues/1023#issuecomment-265344421 +// We also resolve them to make sure all tools using them work consistently. +const appDirectory = fs.realpathSync(process.cwd()) +process.env.NODE_PATH = (process.env.NODE_PATH || "") + .split(path.delimiter) + .filter((folder) => folder && !path.isAbsolute(folder)) + .map((folder) => path.resolve(appDirectory, folder)) + .join(path.delimiter) + +// Grab NODE_ENV and REACT_APP_* environment variables and prepare them to be +// injected into the application via DefinePlugin in webpack configuration. +const REACT_APP = /^REACT_APP_/i + +function getClientEnvironment(publicUrl) { + const raw = Object.keys(process.env) + .filter((key) => REACT_APP.test(key)) + .reduce( + (env, key) => { + env[key] = process.env[key] + return env + }, + { + // Useful for determining whether we’re running in production mode. + // Most importantly, it switches React into the correct mode. + NODE_ENV: process.env.NODE_ENV || "development", + // Useful for resolving the correct path to static assets in `public`. + // For example, . + // This should only be used as an escape hatch. Normally you would put + // images into the `src` and `import` them in code to get their paths. + PUBLIC_URL: publicUrl, + // We support configuring the sockjs pathname during development. + // These settings let a developer run multiple simultaneous projects. + // They are used as the connection `hostname`, `pathname` and `port` + // in webpackHotDevClient. They are used as the `sockHost`, `sockPath` + // and `sockPort` options in webpack-dev-server. + WDS_SOCKET_HOST: process.env.WDS_SOCKET_HOST, + WDS_SOCKET_PATH: process.env.WDS_SOCKET_PATH, + WDS_SOCKET_PORT: process.env.WDS_SOCKET_PORT, + // Whether or not react-refresh is enabled. + // react-refresh is not 100% stable at this time, + // which is why it's disabled by default. + // It is defined here so it is available in the webpackHotDevClient. + FAST_REFRESH: process.env.FAST_REFRESH !== "false", + } + ) + // Stringify all values so we can feed into webpack DefinePlugin + const stringified = { + "process.env": Object.keys(raw).reduce((env, key) => { + env[key] = JSON.stringify(raw[key]) + return env + }, {}), + } + + return { raw, stringified } +} + +module.exports = getClientEnvironment diff --git a/venus-html5-app/config/getHttpsConfig.js b/venus-html5-app/config/getHttpsConfig.js new file mode 100644 index 0000000..7efa29a --- /dev/null +++ b/venus-html5-app/config/getHttpsConfig.js @@ -0,0 +1,56 @@ +const fs = require("fs") +const path = require("path") +const crypto = require("crypto") +const chalk = require("react-dev-utils/chalk") +const paths = require("./paths") + +// Ensure the certificate and key provided are valid and if not +// throw an easy to debug error +function validateKeyAndCerts({ cert, key, keyFile, crtFile }) { + let encrypted + try { + // publicEncrypt will throw an error with an invalid cert + encrypted = crypto.publicEncrypt(cert, Buffer.from("test")) + } catch (err) { + throw new Error(`The certificate "${chalk.yellow(crtFile)}" is invalid.\n${err.message}`) + } + + try { + // privateDecrypt will throw an error with an invalid key + crypto.privateDecrypt(key, encrypted) + } catch (err) { + throw new Error(`The certificate key "${chalk.yellow(keyFile)}" is invalid.\n${err.message}`) + } +} + +// Read file and throw an error if it doesn't exist +function readEnvFile(file, type) { + if (!fs.existsSync(file)) { + throw new Error( + `You specified ${chalk.cyan(type)} in your env, but the file "${chalk.yellow(file)}" can't be found.` + ) + } + return fs.readFileSync(file) +} + +// Get the https config +// Return cert files if provided in env, otherwise just true or false +function getHttpsConfig() { + const { SSL_CRT_FILE, SSL_KEY_FILE, HTTPS } = process.env + const isHttps = HTTPS === "true" + + if (isHttps && SSL_CRT_FILE && SSL_KEY_FILE) { + const crtFile = path.resolve(paths.appPath, SSL_CRT_FILE) + const keyFile = path.resolve(paths.appPath, SSL_KEY_FILE) + const config = { + cert: readEnvFile(crtFile, "SSL_CRT_FILE"), + key: readEnvFile(keyFile, "SSL_KEY_FILE"), + } + + validateKeyAndCerts({ ...config, keyFile, crtFile }) + return config + } + return isHttps +} + +module.exports = getHttpsConfig diff --git a/venus-html5-app/config/jest/babelTransform.js b/venus-html5-app/config/jest/babelTransform.js new file mode 100644 index 0000000..9bf17b0 --- /dev/null +++ b/venus-html5-app/config/jest/babelTransform.js @@ -0,0 +1,27 @@ +const babelJest = require("babel-jest") + +const hasJsxRuntime = (() => { + if (process.env.DISABLE_NEW_JSX_TRANSFORM === "true") { + return false + } + + try { + require.resolve("react/jsx-runtime") + return true + } catch (e) { + return false + } +})() + +module.exports = babelJest.createTransformer({ + presets: [ + [ + require.resolve("babel-preset-react-app"), + { + runtime: hasJsxRuntime ? "automatic" : "classic", + }, + ], + ], + babelrc: false, + configFile: false, +}) diff --git a/venus-html5-app/config/jest/cssTransform.js b/venus-html5-app/config/jest/cssTransform.js new file mode 100644 index 0000000..a97276c --- /dev/null +++ b/venus-html5-app/config/jest/cssTransform.js @@ -0,0 +1,12 @@ +// This is a custom Jest transformer turning style imports into empty objects. +// http://facebook.github.io/jest/docs/en/webpack.html + +module.exports = { + process() { + return "module.exports = {};" + }, + getCacheKey() { + // The output is always the same. + return "cssTransform" + }, +} diff --git a/venus-html5-app/config/jest/fileTransform.js b/venus-html5-app/config/jest/fileTransform.js new file mode 100644 index 0000000..d720655 --- /dev/null +++ b/venus-html5-app/config/jest/fileTransform.js @@ -0,0 +1,38 @@ +const path = require("path") +const camelcase = require("camelcase") + +// This is a custom Jest transformer turning file imports into filenames. +// http://facebook.github.io/jest/docs/en/webpack.html + +module.exports = { + process(src, filename) { + const assetFilename = JSON.stringify(path.basename(filename)) + + if (filename.match(/\.svg$/)) { + // Based on how SVGR generates a component name: + // https://github.com/smooth-code/svgr/blob/01b194cf967347d43d4cbe6b434404731b87cf27/packages/core/src/state.js#L6 + const pascalCaseFilename = camelcase(path.parse(filename).name, { + pascalCase: true, + }) + const componentName = `Svg${pascalCaseFilename}` + return `const React = require('react'); + module.exports = { + __esModule: true, + default: ${assetFilename}, + ReactComponent: React.forwardRef(function ${componentName}(props, ref) { + return { + $$typeof: Symbol.for('react.element'), + type: 'svg', + ref: ref, + key: null, + props: Object.assign({}, props, { + children: ${assetFilename} + }) + }; + }), + };` + } + + return `module.exports = ${assetFilename};` + }, +} diff --git a/venus-html5-app/config/modules.js b/venus-html5-app/config/modules.js new file mode 100644 index 0000000..6fcf390 --- /dev/null +++ b/venus-html5-app/config/modules.js @@ -0,0 +1,132 @@ +const fs = require("fs") +const path = require("path") +const paths = require("./paths") +const chalk = require("react-dev-utils/chalk") +const resolve = require("resolve") + +/** + * Get additional module paths based on the baseUrl of a compilerOptions object. + * + * @param {Object} options + */ +function getAdditionalModulePaths(options = {}) { + const baseUrl = options.baseUrl + + if (!baseUrl) { + return "" + } + + const baseUrlResolved = path.resolve(paths.appPath, baseUrl) + + // We don't need to do anything if `baseUrl` is set to `node_modules`. This is + // the default behavior. + if (path.relative(paths.appNodeModules, baseUrlResolved) === "") { + return null + } + + // Allow the user set the `baseUrl` to `appSrc`. + if (path.relative(paths.appSrc, baseUrlResolved) === "") { + return [paths.appSrc] + } + + // If the path is equal to the root directory we ignore it here. + // We don't want to allow importing from the root directly as source files are + // not transpiled outside of `src`. We do allow importing them with the + // absolute path (e.g. `src/Components/Button.js`) but we set that up with + // an alias. + if (path.relative(paths.appPath, baseUrlResolved) === "") { + return null + } + + // Otherwise, throw an error. + throw new Error( + chalk.red.bold( + "Your project's `baseUrl` can only be set to `src` or `node_modules`." + + " Create React App does not support other values at this time." + ) + ) +} + +/** + * Get webpack aliases based on the baseUrl of a compilerOptions object. + * + * @param {*} options + */ +function getWebpackAliases(options = {}) { + const baseUrl = options.baseUrl + + if (!baseUrl) { + return {} + } + + const baseUrlResolved = path.resolve(paths.appPath, baseUrl) + + if (path.relative(paths.appPath, baseUrlResolved) === "") { + return { + src: paths.appSrc, + } + } +} + +/** + * Get jest aliases based on the baseUrl of a compilerOptions object. + * + * @param {*} options + */ +function getJestAliases(options = {}) { + const baseUrl = options.baseUrl + + if (!baseUrl) { + return {} + } + + const baseUrlResolved = path.resolve(paths.appPath, baseUrl) + + if (path.relative(paths.appPath, baseUrlResolved) === "") { + return { + "^src/(.*)$": "/src/$1", + } + } +} + +function getModules() { + // Check if TypeScript is setup + const hasTsConfig = fs.existsSync(paths.appTsConfig) + const hasJsConfig = fs.existsSync(paths.appJsConfig) + + if (hasTsConfig && hasJsConfig) { + throw new Error( + "You have both a tsconfig.json and a jsconfig.json. If you are using TypeScript please remove your jsconfig.json file." + ) + } + + let config + + // If there's a tsconfig.json we assume it's a + // TypeScript project and set up the config + // based on tsconfig.json + if (hasTsConfig) { + const ts = require(resolve.sync("typescript", { + basedir: paths.appNodeModules, + })) + config = ts.readConfigFile(paths.appTsConfig, ts.sys.readFile).config + // Otherwise we'll check if there is jsconfig.json + // for non TS projects. + } else if (hasJsConfig) { + config = require(paths.appJsConfig) + } + + config = config || {} + const options = config.compilerOptions || {} + + const additionalModulePaths = getAdditionalModulePaths(options) + + return { + additionalModulePaths: additionalModulePaths, + webpackAliases: getWebpackAliases(options), + jestAliases: getJestAliases(options), + hasTsConfig, + } +} + +module.exports = getModules() diff --git a/venus-html5-app/config/paths.js b/venus-html5-app/config/paths.js new file mode 100644 index 0000000..b944c9b --- /dev/null +++ b/venus-html5-app/config/paths.js @@ -0,0 +1,69 @@ +const path = require("path") +const fs = require("fs") +const getPublicUrlOrPath = require("react-dev-utils/getPublicUrlOrPath") + +// Make sure any symlinks in the project folder are resolved: +// https://github.com/facebook/create-react-app/issues/637 +const appDirectory = fs.realpathSync(process.cwd()) +const resolveApp = (relativePath) => path.resolve(appDirectory, relativePath) + +// We use `PUBLIC_URL` environment variable or "homepage" field to infer +// "public path" at which the app is served. +// webpack needs to know it to put the right + + \ No newline at end of file diff --git a/venus-html5-app/public/icons/selectors/dot-selected-vert.svg b/venus-html5-app/public/icons/selectors/dot-selected-vert.svg new file mode 100644 index 0000000..199ccd9 --- /dev/null +++ b/venus-html5-app/public/icons/selectors/dot-selected-vert.svg @@ -0,0 +1,3 @@ + + + diff --git a/venus-html5-app/public/icons/selectors/dot-selected.svg b/venus-html5-app/public/icons/selectors/dot-selected.svg new file mode 100644 index 0000000..0b3419f --- /dev/null +++ b/venus-html5-app/public/icons/selectors/dot-selected.svg @@ -0,0 +1,3 @@ + + + diff --git a/venus-html5-app/public/icons/selectors/dot.svg b/venus-html5-app/public/icons/selectors/dot.svg new file mode 100644 index 0000000..a4eecf8 --- /dev/null +++ b/venus-html5-app/public/icons/selectors/dot.svg @@ -0,0 +1,3 @@ + + + diff --git a/venus-html5-app/public/icons/selectors/selector-down-blue.svg b/venus-html5-app/public/icons/selectors/selector-down-blue.svg new file mode 100644 index 0000000..b0f8e20 --- /dev/null +++ b/venus-html5-app/public/icons/selectors/selector-down-blue.svg @@ -0,0 +1,4 @@ + + + + diff --git a/venus-html5-app/public/icons/selectors/selector-down.svg b/venus-html5-app/public/icons/selectors/selector-down.svg new file mode 100644 index 0000000..008d790 --- /dev/null +++ b/venus-html5-app/public/icons/selectors/selector-down.svg @@ -0,0 +1,4 @@ + + + + diff --git a/venus-html5-app/public/icons/selectors/selector-left-blue.svg b/venus-html5-app/public/icons/selectors/selector-left-blue.svg new file mode 100644 index 0000000..6aa679f --- /dev/null +++ b/venus-html5-app/public/icons/selectors/selector-left-blue.svg @@ -0,0 +1,4 @@ + + + + diff --git a/venus-html5-app/public/icons/selectors/selector-left.svg b/venus-html5-app/public/icons/selectors/selector-left.svg new file mode 100644 index 0000000..9c17835 --- /dev/null +++ b/venus-html5-app/public/icons/selectors/selector-left.svg @@ -0,0 +1,6 @@ + + + + diff --git a/venus-html5-app/public/icons/selectors/selector-right-blue.svg b/venus-html5-app/public/icons/selectors/selector-right-blue.svg new file mode 100644 index 0000000..ceca74d --- /dev/null +++ b/venus-html5-app/public/icons/selectors/selector-right-blue.svg @@ -0,0 +1,4 @@ + + + + diff --git a/venus-html5-app/public/icons/selectors/selector-right.svg b/venus-html5-app/public/icons/selectors/selector-right.svg new file mode 100644 index 0000000..efc4003 --- /dev/null +++ b/venus-html5-app/public/icons/selectors/selector-right.svg @@ -0,0 +1,6 @@ + + + + diff --git a/venus-html5-app/public/icons/selectors/selector-up-blue.svg b/venus-html5-app/public/icons/selectors/selector-up-blue.svg new file mode 100644 index 0000000..201f920 --- /dev/null +++ b/venus-html5-app/public/icons/selectors/selector-up-blue.svg @@ -0,0 +1,4 @@ + + + + diff --git a/venus-html5-app/public/icons/selectors/selector-up.svg b/venus-html5-app/public/icons/selectors/selector-up.svg new file mode 100644 index 0000000..031a442 --- /dev/null +++ b/venus-html5-app/public/icons/selectors/selector-up.svg @@ -0,0 +1,4 @@ + + + + diff --git a/venus-html5-app/public/index.html b/venus-html5-app/public/index.html new file mode 100644 index 0000000..761905d --- /dev/null +++ b/venus-html5-app/public/index.html @@ -0,0 +1,99 @@ + + + + + Victron + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + diff --git a/venus-html5-app/public/radiator.html b/venus-html5-app/public/radiator.html new file mode 100644 index 0000000..5f3365b --- /dev/null +++ b/venus-html5-app/public/radiator.html @@ -0,0 +1,91 @@ + + + + + +
+ + + +
+ + + + + + + + + diff --git a/venus-html5-app/public/sounds/mixkit-bell-notification-933.wav b/venus-html5-app/public/sounds/mixkit-bell-notification-933.wav new file mode 100644 index 0000000..7e3d6ad Binary files /dev/null and b/venus-html5-app/public/sounds/mixkit-bell-notification-933.wav differ diff --git a/venus-html5-app/screenshot.png b/venus-html5-app/screenshot.png new file mode 100644 index 0000000..8b0f98c Binary files /dev/null and b/venus-html5-app/screenshot.png differ diff --git a/venus-html5-app/scripts/build.js b/venus-html5-app/scripts/build.js new file mode 100644 index 0000000..b2290ad --- /dev/null +++ b/venus-html5-app/scripts/build.js @@ -0,0 +1,191 @@ +// Do this as the first thing so that any code reading it knows the right env. +process.env.BABEL_ENV = "production" +process.env.NODE_ENV = "production" +process.env.GENERATE_SOURCEMAP = "false" + +// Makes the script crash on unhandled rejections instead of silently +// ignoring them. In the future, promise rejections that are not handled will +// terminate the Node.js process with a non-zero exit code. +process.on("unhandledRejection", (err) => { + throw err +}) + +// Ensure environment variables are read. +require("../config/env") + +const path = require("path") +const chalk = require("react-dev-utils/chalk") +const fs = require("fs-extra") +const bfj = require("bfj") +const webpack = require("webpack") +const configFactory = require("../config/webpack.config") +const paths = require("../config/paths") +const checkRequiredFiles = require("react-dev-utils/checkRequiredFiles") +const formatWebpackMessages = require("react-dev-utils/formatWebpackMessages") +const printHostingInstructions = require("react-dev-utils/printHostingInstructions") +const FileSizeReporter = require("react-dev-utils/FileSizeReporter") +const printBuildError = require("react-dev-utils/printBuildError") + +const measureFileSizesBeforeBuild = FileSizeReporter.measureFileSizesBeforeBuild +const printFileSizesAfterBuild = FileSizeReporter.printFileSizesAfterBuild +const useYarn = fs.existsSync(paths.yarnLockFile) + +// These sizes are pretty large. We'll warn for bundles exceeding them. +const WARN_AFTER_BUNDLE_GZIP_SIZE = 512 * 1024 +const WARN_AFTER_CHUNK_GZIP_SIZE = 1024 * 1024 + +const isInteractive = process.stdout.isTTY + +// Warn and crash if required files are missing +if (!checkRequiredFiles([paths.appHtml, paths.appIndexJs])) { + process.exit(1) +} + +const argv = process.argv.slice(2) +const writeStatsJson = argv.indexOf("--stats") !== -1 + +// Generate configuration +const config = configFactory("production") + +// We require that you explicitly set browsers and do not fall back to +// browserslist defaults. +const { checkBrowsers } = require("react-dev-utils/browsersHelper") +checkBrowsers(paths.appPath, isInteractive) + .then(() => { + // First, read the current file sizes in build directory. + // This lets us display how much they changed later. + return measureFileSizesBeforeBuild(paths.appBuild) + }) + .then((previousFileSizes) => { + // Remove all content but keep the directory so that + // if you're in it, you don't end up in Trash + fs.emptyDirSync(paths.appBuild) + // Merge with the public folder + copyPublicFolder() + // Start the webpack build + return build(previousFileSizes) + }) + .then( + ({ stats, previousFileSizes, warnings }) => { + if (warnings.length) { + console.log(chalk.yellow("Compiled with warnings.\n")) + console.log(warnings.join("\n\n")) + console.log( + "\nSearch for the " + chalk.underline(chalk.yellow("keywords")) + " to learn more about each warning." + ) + console.log("To ignore, add " + chalk.cyan("// eslint-disable-next-line") + " to the line before.\n") + } else { + console.log(chalk.green("Compiled successfully.\n")) + } + + console.log("File sizes after gzip:\n") + printFileSizesAfterBuild( + stats, + previousFileSizes, + paths.appBuild, + WARN_AFTER_BUNDLE_GZIP_SIZE, + WARN_AFTER_CHUNK_GZIP_SIZE + ) + console.log() + + const appPackage = require(paths.appPackageJson) + const publicUrl = paths.publicUrlOrPath + const publicPath = config.output.publicPath + const buildFolder = path.relative(process.cwd(), paths.appBuild) + printHostingInstructions(appPackage, publicUrl, publicPath, buildFolder, useYarn) + }, + (err) => { + const tscCompileOnError = process.env.TSC_COMPILE_ON_ERROR === "true" + if (tscCompileOnError) { + console.log( + chalk.yellow( + "Compiled with the following type errors (you may want to check these before deploying your app):\n" + ) + ) + printBuildError(err) + } else { + console.log(chalk.red("Failed to compile.\n")) + printBuildError(err) + process.exit(1) + } + } + ) + .catch((err) => { + if (err && err.message) { + console.log(err.message) + } + process.exit(1) + }) + +// Create the production build and print the deployment instructions. +function build(previousFileSizes) { + console.log("Creating an optimized production build...") + + const compiler = webpack(config) + return new Promise((resolve, reject) => { + compiler.run((err, stats) => { + let messages + if (err) { + if (!err.message) { + return reject(err) + } + + let errMessage = err.message + + // Add additional information for postcss errors + if (Object.prototype.hasOwnProperty.call(err, "postcssNode")) { + errMessage += "\nCompileError: Begins at CSS selector " + err["postcssNode"].selector + } + + messages = formatWebpackMessages({ + errors: [errMessage], + warnings: [], + }) + } else { + messages = formatWebpackMessages(stats.toJson({ all: false, warnings: true, errors: true })) + } + if (messages.errors.length) { + // Only keep the first error. Others are often indicative + // of the same problem, but confuse the reader with noise. + if (messages.errors.length > 1) { + messages.errors.length = 1 + } + return reject(new Error(messages.errors.join("\n\n"))) + } + if ( + process.env.CI && + (typeof process.env.CI !== "string" || process.env.CI.toLowerCase() !== "false") && + messages.warnings.length + ) { + console.log( + chalk.yellow( + "\nTreating warnings as errors because process.env.CI = true.\n" + "Most CI servers set it automatically.\n" + ) + ) + return reject(new Error(messages.warnings.join("\n\n"))) + } + + const resolveArgs = { + stats, + previousFileSizes, + warnings: messages.warnings, + } + + if (writeStatsJson) { + return bfj + .write(paths.appBuild + "/bundle-stats.json", stats.toJson()) + .then(() => resolve(resolveArgs)) + .catch((error) => reject(new Error(error))) + } + + return resolve(resolveArgs) + }) + }) +} + +function copyPublicFolder() { + fs.copySync(paths.appPublic, paths.appBuild, { + dereference: true, + filter: (file) => file !== paths.appHtml, + }) +} diff --git a/venus-html5-app/scripts/serve-local.js b/venus-html5-app/scripts/serve-local.js new file mode 100644 index 0000000..86b1867 --- /dev/null +++ b/venus-html5-app/scripts/serve-local.js @@ -0,0 +1,68 @@ +const http = require("http") +const fs = require("fs") +const path = require("path") + +const PORT = parseInt(process.env.SERVE_PORT, 10) || 3001 +const DIST_DIR = path.resolve(__dirname, "..", "dist") +const APP_PREFIX = "/app" + +const MIME_TYPES = { + ".html": "text/html", + ".js": "application/javascript", + ".css": "text/css", + ".json": "application/json", + ".png": "image/png", + ".jpg": "image/jpeg", + ".svg": "image/svg+xml", + ".ico": "image/x-icon", + ".woff": "font/woff", + ".woff2": "font/woff2", + ".ttf": "font/ttf", + ".map": "application/json", + ".txt": "text/plain", + ".webmanifest": "application/manifest+json", +} + +const server = http.createServer((req, res) => { + let urlPath = req.url.split("?")[0] + + if (urlPath === "/") { + res.writeHead(302, { Location: "/app/" }) + res.end() + return + } + + if (!urlPath.startsWith(APP_PREFIX)) { + res.writeHead(404) + res.end("Not Found") + return + } + + let filePath = urlPath.slice(APP_PREFIX.length) || "/index.html" + if (filePath === "/") filePath = "/index.html" + + const fullPath = path.join(DIST_DIR, filePath) + + if (!fullPath.startsWith(DIST_DIR)) { + res.writeHead(403) + res.end("Forbidden") + return + } + + fs.stat(fullPath, (err, stats) => { + if (!err && stats.isFile()) { + const ext = path.extname(fullPath) + const contentType = MIME_TYPES[ext] || "application/octet-stream" + res.writeHead(200, { "Content-Type": contentType }) + fs.createReadStream(fullPath).pipe(res) + } else { + const indexPath = path.join(DIST_DIR, "index.html") + res.writeHead(200, { "Content-Type": "text/html" }) + fs.createReadStream(indexPath).pipe(res) + } + }) +}) + +server.listen(PORT, "0.0.0.0", () => { + console.log(`Serving venus-html5-app on http://0.0.0.0:${PORT}/app/`) +}) diff --git a/venus-html5-app/scripts/serve.sh b/venus-html5-app/scripts/serve.sh new file mode 100755 index 0000000..26413a4 --- /dev/null +++ b/venus-html5-app/scripts/serve.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +APP_DIR="$SCRIPT_DIR/.." + +find "$APP_DIR/dist" -name \*.gz | xargs gunzip 2>/dev/null + +node "$SCRIPT_DIR/serve-local.js" diff --git a/venus-html5-app/scripts/start.js b/venus-html5-app/scripts/start.js new file mode 100644 index 0000000..83d7855 --- /dev/null +++ b/venus-html5-app/scripts/start.js @@ -0,0 +1,134 @@ +// Do this as the first thing so that any code reading it knows the right env. +process.env.BABEL_ENV = "development" +process.env.NODE_ENV = "development" +process.env.ESLINT_NO_DEV_ERRORS = true + +// Makes the script crash on unhandled rejections instead of silently +// ignoring them. In the future, promise rejections that are not handled will +// terminate the Node.js process with a non-zero exit code. +process.on("unhandledRejection", (err) => { + throw err +}) + +// Ensure environment variables are read. +require("../config/env") + +const fs = require("fs") +const chalk = require("react-dev-utils/chalk") +const webpack = require("webpack") +const WebpackDevServer = require("webpack-dev-server") +const clearConsole = require("react-dev-utils/clearConsole") +const checkRequiredFiles = require("react-dev-utils/checkRequiredFiles") +const { choosePort, createCompiler, prepareProxy, prepareUrls } = require("react-dev-utils/WebpackDevServerUtils") +const openBrowser = require("react-dev-utils/openBrowser") +const semver = require("semver") +const paths = require("../config/paths") +const configFactory = require("../config/webpack.config") +const createDevServerConfig = require("../config/webpackDevServer.config") +const getClientEnvironment = require("../config/env") +const react = require(require.resolve("react", { paths: [paths.appPath] })) + +const env = getClientEnvironment(paths.publicUrlOrPath.slice(0, -1)) +const useYarn = fs.existsSync(paths.yarnLockFile) +const isInteractive = process.stdout.isTTY + +// Warn and crash if required files are missing +if (!checkRequiredFiles([paths.appHtml, paths.appIndexJs])) { + process.exit(1) +} + +// Tools like Cloud9 rely on this. +const DEFAULT_PORT = parseInt(process.env.PORT, 10) || 3000 +const HOST = process.env.HOST || "0.0.0.0" + +if (process.env.HOST) { + console.log( + chalk.cyan(`Attempting to bind to HOST environment variable: ${chalk.yellow(chalk.bold(process.env.HOST))}`) + ) + console.log(`If this was unintentional, check that you haven't mistakenly set it in your shell.`) + console.log(`Learn more here: ${chalk.yellow("https://cra.link/advanced-config")}`) + console.log() +} + +// We require that you explicitly set browsers and do not fall back to +// browserslist defaults. +const { checkBrowsers } = require("react-dev-utils/browsersHelper") +checkBrowsers(paths.appPath, isInteractive) + .then(() => { + // We attempt to use the default port but if it is busy, we offer the user to + // run on a different port. `choosePort()` Promise resolves to the next free port. + return choosePort(HOST, DEFAULT_PORT) + }) + .then((port) => { + if (port == null) { + // We have not found a port. + return + } + + const config = configFactory("development") + const protocol = process.env.HTTPS === "true" ? "https" : "http" + const appName = require(paths.appPackageJson).name + + const useTypeScript = fs.existsSync(paths.appTsConfig) + const tscCompileOnError = process.env.TSC_COMPILE_ON_ERROR === "true" + const urls = prepareUrls(protocol, HOST, port, paths.publicUrlOrPath.slice(0, -1)) + const devSocket = { + warnings: (warnings) => devServer.sockWrite(devServer.sockets, "warnings", warnings), + errors: (errors) => devServer.sockWrite(devServer.sockets, "errors", errors), + } + // Create a webpack compiler that is configured with custom messages. + //todo: replace with original webpack compiler + // const compiler = webpack(config); + const compiler = createCompiler({ + appName, + config, + devSocket, + urls, + useYarn, + useTypeScript, + tscCompileOnError, + webpack, + }) + // Load proxy config + const proxySetting = require(paths.appPackageJson).proxy + const proxyConfig = prepareProxy(proxySetting, paths.appPublic, paths.publicUrlOrPath) + // Serve webpack assets generated by the compiler over a web server. + const serverConfig = createDevServerConfig(proxyConfig, urls.lanUrlForConfig) + const devServer = new WebpackDevServer({...serverConfig.devServer}, compiler) + // Launch WebpackDevServer. + devServer.listen(port, HOST, (err) => { + if (err) { + return console.log(err) + } + if (isInteractive) { + clearConsole() + } + + if (env.raw.FAST_REFRESH && semver.lt(react.version, "16.10.0")) { + console.log(chalk.yellow(`Fast Refresh requires React 16.10 or higher. You are using React ${react.version}.`)) + } + + console.log(chalk.cyan("Starting the development server...\n")) + openBrowser(urls.localUrlForBrowser) + }) + ;["SIGINT", "SIGTERM"].forEach(function (sig) { + process.on(sig, function () { + devServer.close() + process.exit() + }) + }) + + if (process.env.CI !== "true") { + // Gracefully exit when stdin ends + process.stdin.on("end", function () { + devServer.close() + process.exit() + }) + } + }) + .catch((err) => { + if (err && err.message) { + console.log(err.message) + } + process.exit(1) + }) diff --git a/venus-html5-app/scripts/test.js b/venus-html5-app/scripts/test.js new file mode 100644 index 0000000..0df2b03 --- /dev/null +++ b/venus-html5-app/scripts/test.js @@ -0,0 +1,45 @@ +// Do this as the first thing so that any code reading it knows the right env. +process.env.BABEL_ENV = "test" +process.env.NODE_ENV = "test" +process.env.PUBLIC_URL = "" + +// Makes the script crash on unhandled rejections instead of silently +// ignoring them. In the future, promise rejections that are not handled will +// terminate the Node.js process with a non-zero exit code. +process.on("unhandledRejection", (err) => { + throw err +}) + +// Ensure environment variables are read. +require("../config/env") + +const jest = require("jest") +const execSync = require("child_process").execSync +let argv = process.argv.slice(2) + +function isInGitRepository() { + try { + execSync("git rev-parse --is-inside-work-tree", { stdio: "ignore" }) + return true + } catch (e) { + return false + } +} + +function isInMercurialRepository() { + try { + execSync("hg --cwd . root", { stdio: "ignore" }) + return true + } catch (e) { + return false + } +} + +// Watch unless on CI or explicitly running all tests +if (!process.env.CI && argv.indexOf("--watchAll") === -1 && argv.indexOf("--watchAll=false") === -1) { + // https://github.com/facebook/create-react-app/issues/5210 + const hasSourceControl = isInGitRepository() || isInMercurialRepository() + argv.push(hasSourceControl ? "--watch" : "--watchAll") +} + +jest.run(argv) diff --git a/venus-html5-app/serve-root/app b/venus-html5-app/serve-root/app new file mode 120000 index 0000000..375d4b6 --- /dev/null +++ b/venus-html5-app/serve-root/app @@ -0,0 +1 @@ +/home/dev/projects/venus/venus-html5-app/dist \ No newline at end of file diff --git a/venus-html5-app/serve-root/serve.json b/venus-html5-app/serve-root/serve.json new file mode 100644 index 0000000..15b6f78 --- /dev/null +++ b/venus-html5-app/serve-root/serve.json @@ -0,0 +1,5 @@ +{ + "rewrites": [ + { "source": "app/**", "destination": "/app/index.html" } + ] +} diff --git a/venus-html5-app/serve.json b/venus-html5-app/serve.json new file mode 100644 index 0000000..f100b7f --- /dev/null +++ b/venus-html5-app/serve.json @@ -0,0 +1,8 @@ +{ + "public": "./dist", + "rewrites": [ + { "source": "app/:a", "destination": ":a" }, + { "source": "app/:a/:b", "destination": ":a/:b" }, + { "source": "app/:a/:b/:c", "destination": ":a/:b/:c" } + ] +} diff --git a/venus-html5-app/src/app/KVNRV/App.tsx b/venus-html5-app/src/app/KVNRV/App.tsx new file mode 100644 index 0000000..abe53d5 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/App.tsx @@ -0,0 +1,52 @@ +import { useAppStore, useMqtt, useVebus, useVrmStore } from "@victronenergy/mfd-modules" +import { observer } from "mobx-react" +import React, { useEffect } from "react" +import "../../css/index.scss" +import { getLocale } from "react-i18nify" +// TODO: copy this lib under KVNRV folder +import { useVisibleWidgetsStore } from "../MarineApp/modules" + +import { KVNRV } from "./KVNRV" + +export type AppProps = { + protocol: string + host: string + port: number | null + path: string +} + +const App = observer((props: AppProps) => { + const vrmStore = useVrmStore() + const appStore = useAppStore() + const mqtt = useMqtt() + const locale = getLocale() + const visibleWidgetsStore = useVisibleWidgetsStore() + useVebus() + + useEffect(() => { + if (!appStore.remote) { + mqtt.boot(props.protocol, props.host, props.port, props.path) + } else if (appStore.remote && vrmStore?.webhost && vrmStore?.portalId && vrmStore?.siteId) { + mqtt.boot("https", vrmStore.webhost, null, "", true, vrmStore.portalId) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [ + props.protocol, + props.host, + props.port, + appStore.remote, + vrmStore.webhost, + vrmStore.portalId, + vrmStore.siteId, + locale, + ]) + + useEffect(() => { + visibleWidgetsStore.clearVisibleElements() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [locale]) + + return +}) + +export default App diff --git a/venus-html5-app/src/app/KVNRV/KVNRV.tsx b/venus-html5-app/src/app/KVNRV/KVNRV.tsx new file mode 100644 index 0000000..8e2aa05 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/KVNRV.tsx @@ -0,0 +1,49 @@ +import React from "react" +import { Header } from "./components/Header" +import { Connecting, Error, Metrics, MqttUnavailable, RemoteConsole } from "./components/Views" +import { mfdLanguageOptions } from "app/locales/constants" +import { STATUS, useApp, useMqtt, useTheme, useLanguage } from "@victronenergy/mfd-modules" +import { VIEWS } from "./utils/constants" +import RemoteLogin from "./components/Views/RemoteLogin" +import { AppProps } from "./App" +import { observer } from "mobx-react" + +export const KVNRV = observer((props: AppProps) => { + const { darkMode } = useTheme() + // subscribe to language + useLanguage(mfdLanguageOptions) + const appStore = useApp() + const { error, status } = useMqtt() + + return ( +
+
+ {(() => { + switch (appStore?.page) { + case VIEWS.CONSOLE: + return ( + { + appStore.setPage(VIEWS.METRICS) + }} + host={props.host} + /> + ) + default: + case VIEWS.METRICS: + if (error && status === STATUS.OFFLINE) { + return + } else if (error && status !== STATUS.CONNECTING) { + return + } else if (status === STATUS.CONNECTING) { + return + } + + return + case VIEWS.LOGIN: + return + } + })()} +
+ ) +}) diff --git a/venus-html5-app/src/app/KVNRV/components/AcLoads/AcLoads.tsx b/venus-html5-app/src/app/KVNRV/components/AcLoads/AcLoads.tsx new file mode 100644 index 0000000..45748a5 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/AcLoads/AcLoads.tsx @@ -0,0 +1,106 @@ +import React, { useEffect, useState } from "react" + +import { useAcLoads, useAcMode, useSystemState } from "@victronenergy/mfd-modules" +import { Card, SIZE_NARROW, SIZE_SHORT } from "../../../components/Card" +import { normalizePower } from "../../utils/helpers" +import { AC_CONF, AC_MODE, CRITICAL_MULTIPLIER, WidgetConfiguration } from "../../utils/constants" +import NumericValue from "../../../components/NumericValue" +import { NotAvailable } from "../NotAvailable" +import { useSendUpdate } from "../../modules" +import { Translate } from "react-i18nify" +import { observer } from "mobx-react" +import { KVNGauge } from "../KVNGauge" + +const inverterPeakPower = 3000 +const inverterContinuousPower = 2000 +const inverterCautionPower = 1400 + +const acLimit = (inverterMode: number, inPowerLimit: number, systemState: number, voltage: number) => { + const outPowerLimit = 30 * 120 + let barMax = 0, + overload = 0, + caution = 0 + + // Inverter Only - only multi contribution + if (inverterMode === 2 || systemState === 9) { + barMax = inverterPeakPower + overload = inverterContinuousPower + caution = inverterCautionPower + } + // Charger Only - only AC input contribution + else if (inverterMode === 1) { + barMax = inPowerLimit * voltage + overload = inPowerLimit * voltage + caution = inPowerLimit * voltage * 0.8 + } + // On - AC input + multi contribution + else if (inverterMode === 3 && systemState >= 3) { + barMax = inPowerLimit + inverterPeakPower + overload = inPowerLimit + inverterContinuousPower + caution = inPowerLimit + inverterCautionPower + } + // inverter is off or undefined - no AC output + else { + barMax = 1 + overload = 1 + caution = 1 + } + // apply system output limit + if (overload > outPowerLimit) { + caution = outPowerLimit * 0.8 + overload = outPowerLimit + barMax = outPowerLimit + } + + barMax = barMax * CRITICAL_MULTIPLIER + let green = caution / barMax + let yellow = overload / barMax - green + let red = 1 - (yellow + green) + + return { ...AC_CONF, MAX: barMax, THRESHOLDS: [green, yellow, red] } as WidgetConfiguration +} + +export const AcLoads = observer(() => { + const { systemState } = useSystemState() + const { mode, limit } = useAcMode() + const { current, voltage, power, frequency } = useAcLoads() + const [config, setConfig] = useState(AC_CONF) + + const inLimit = Number(limit) + const inMode = Number(mode) + + useEffect(() => { + setConfig(acLimit(inMode, isNaN(inLimit) ? AC_MODE.LIMITS_US[0] : inLimit, systemState, voltage ? voltage[0] : 1)) + }, [inMode, inLimit, systemState, voltage]) + + const normalizedPower = normalizePower(power && power[0] ? power[0] : 0, config.MAX) + useSendUpdate(normalizedPower, config, "AC Loads") + + return ( + } size={[SIZE_SHORT, SIZE_NARROW]}> +
+ {!isNaN(inMode) && inMode !== AC_MODE.MODES.OFF && power ? ( + <> + + +
+
+ +
+
+ +
+
+ +
+
+ + ) : ( + + )} +
+ + ) +}) + +export default AcLoads diff --git a/venus-html5-app/src/app/KVNRV/components/AcLoads/index.js b/venus-html5-app/src/app/KVNRV/components/AcLoads/index.js new file mode 100644 index 0000000..fdc662a --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/AcLoads/index.js @@ -0,0 +1 @@ +export { default } from "./AcLoads" diff --git a/venus-html5-app/src/app/KVNRV/components/AcMode/AcMode.scss b/venus-html5-app/src/app/KVNRV/components/AcMode/AcMode.scss new file mode 100644 index 0000000..7290cff --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/AcMode/AcMode.scss @@ -0,0 +1,89 @@ +@import "../../css/variables"; + +.ac_mode { + flex-direction: column; + text-align: center; + width: auto; + + .name { + font-size: 1rem; + } + + .info-bar__cell { + padding: 0.15rem 0.3rem; + } + +} + +.ac_mode_modal { + flex-direction: column; + align-items: center; + width: 100%; + + .name:nth-child(n+2) { + margin-top: 0.9rem; + } + + .ac_mode_modal__group { + width: inherit; + display: flex; + justify-content: space-between; + flex-wrap: wrap; + + .ac_mode_modal__button { + flex-grow: 1; + margin: 0.3rem; + border-radius: 5px; + border: none; + background-color: $color-lightgray; + padding: 0.7rem; + font-size: 0.9rem; + + &.success { + color: $color-white; + } + } + } +} + +.ac-container { + height: 100%; + display: flex; + width: 100%; + .gauge { + width: 40%; + height: 90%; + } + .ac-actions { + width: 60%; + } + .action { + width: 100%; + display: flex; + flex-direction: column; + align-items: flex-start; + + &:nth-child(2) { + margin-top: 0.4rem; + } + & > .name { + color: $color-gray-600; + font-family: MuseoSans; + font-size: 0.8rem; + letter-spacing: 0; + line-height: 1.1rem; + } + & > .btn { + margin-top: 0.3rem; + } + + &-btn-content { + display: flex; + width: 100%; + & > img { + width: 0.5rem; + margin-left: 0.5rem; + } + } + } +} \ No newline at end of file diff --git a/venus-html5-app/src/app/KVNRV/components/AcMode/AcMode.tsx b/venus-html5-app/src/app/KVNRV/components/AcMode/AcMode.tsx new file mode 100644 index 0000000..1e09df1 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/AcMode/AcMode.tsx @@ -0,0 +1,108 @@ +import React, { useState } from "react" + +import { Card, SIZE_WIDE, SIZE_SHORT } from "../../../components/Card" +import { AC_MODE, CRITICAL_MULTIPLIER, SHORE_POWER_CONF } from "../../utils/constants" +import NumericValue, { formatNumber } from "../../../components/NumericValue" +import { AcModeModal } from "./AcModeModal" + +import "./AcMode.scss" +import { useAcMode, useActiveInValues, useActiveSource } from "@victronenergy/mfd-modules" +import { useSendUpdate } from "../../modules" +import { normalizePower } from "../../utils/helpers" +import { observer } from "mobx-react" +import { KVNGauge } from "../KVNGauge" +import { Translate } from "react-i18nify" +import RIcon from "../../../../images/icons/R.svg" + +export const acModeFormatter = (value: number) => { + switch (value) { + case AC_MODE.MODES.ON: + return "on" + case AC_MODE.MODES.OFF: + return "off" + case AC_MODE.MODES.CHARGER_ONLY: + return "chargerOnly" + case AC_MODE.MODES.INVERTER_ONLY: + return "inverterOnly" + default: + return "emptyBar" + } +} + +export const AcMode = observer(() => { + const { mode, limit, productId, updateMode, updateLimit } = useAcMode() + const { activeInput } = useActiveSource() + const { current, frequency, voltage, power } = useActiveInValues() + + const inLimit = parseFloat(limit ?? "0") + const powerMax = inLimit * (voltage && voltage[0] ? voltage[0] : 1) * CRITICAL_MULTIPLIER + const normalizedPower = normalizePower(power && power[0] ? power[0] : 0, powerMax) + useSendUpdate(normalizedPower, SHORE_POWER_CONF, "Shore Power") + + const [modalOpen, setModalOpen] = useState(false) + + return ( +
+ } size={[SIZE_WIDE, SIZE_SHORT]}> +
+
+
+ + + + +
+
+ + + + +
+
+
+ {(activeInput === 0 || activeInput === 1) && power ? ( + + ) : ( +
+ +
+ )} + +
+
+ +
+
+ +
+
+ +
+
+
+
+ + {modalOpen && ( + setModalOpen(false)} + productId={productId} + updateMode={updateMode} + updateLimit={updateLimit} + /> + )} + +
+ ) +}) + +export default AcMode diff --git a/venus-html5-app/src/app/KVNRV/components/AcMode/AcModeModal.tsx b/venus-html5-app/src/app/KVNRV/components/AcMode/AcModeModal.tsx new file mode 100644 index 0000000..643071b --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/AcMode/AcModeModal.tsx @@ -0,0 +1,74 @@ +import Modal from "../../../components/Modal" +import { AC_MODE } from "../../utils/constants" +import { acModeFormatter } from "./AcMode" +import Logger from "../../../utils/logger" +import { translate, Translate } from "react-i18nify" + +/** + * - Mask the Product id with `0xFF00` + * - If the result is `0x1900` or `0x2600` it is an EU model (230VAC) + * - If the result is `0x2000` or `0x2700` it is an US model (120VAC) + */ + +const getSuggestedAmperageValuesList = (productId: number) => { + const result = productId & 0xff00 + if (result === 0x1900 || result === 0x2600) { + return AC_MODE.LIMITS_EU + } else if (result === 0x2000 || result === 0x2700) { + return AC_MODE.LIMITS_US + } else { + Logger.warn(`Could not determine amperage US/EU for product id ${productId}`) + return AC_MODE.LIMITS_US + } +} + +type AcModeModalProps = { + productId: number + mode: number + limit?: number + updateMode: Function + updateLimit?: Function + onClose: Function +} + +export const AcModeModal = (props: AcModeModalProps) => { + return ( + +
+
+ +
+
+ {Object.values(AC_MODE.MODES).map((mode) => ( + + ))} +
+ + {props.updateLimit && ( + <> +
+ +
+
+ {Object.values(getSuggestedAmperageValuesList(props.productId)).map((limit) => ( + + ))} +
+ + )} +
+ + ) +} diff --git a/venus-html5-app/src/app/KVNRV/components/AcMode/index.js b/venus-html5-app/src/app/KVNRV/components/AcMode/index.js new file mode 100644 index 0000000..0ac34ab --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/AcMode/index.js @@ -0,0 +1 @@ +export { default } from "./AcMode" diff --git a/venus-html5-app/src/app/KVNRV/components/Battery/Battery.scss b/venus-html5-app/src/app/KVNRV/components/Battery/Battery.scss new file mode 100644 index 0000000..78209c5 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Battery/Battery.scss @@ -0,0 +1,133 @@ +@import "../../css/variables"; + + +$battery-width: 3.5rem; +$battery-height: 5rem; + + +.remaining { + margin-top: 1rem; +} + +.indicator-current { + margin-top: 0.6rem; +} + +.battery { + display: flex; + flex-direction: row; + justify-content: space-between; + + .battery__group { + width: 100%; + display: flex; + flex-direction: row; + padding-right: 1rem; + + .indicator-main { + flex-direction: column; + margin-left: 1rem; + } + + .indicator-right { + margin-left: auto; + } + + &.small { + flex-direction: column; + font-size: 0.9rem; + padding-right: 0; + + .indicator-main--small { + flex-direction: column; + } + } + + } + + .battery__charge { + display: flex; + flex-direction: column; + align-items: center; + + .battery__charge__top { + width: $battery-width*0.5; + height: $battery-height*0.1; + border: $color-gray solid 2px; + border-radius: 3px 3px 0 0; + } + + .battery__charge__body { + width: $battery-width; + height: $battery-height; + border: $color-gray solid 2px; + border-radius: 8px; + padding: 0.35rem 0.35rem 0.35rem 0.35rem; + display: flex; + flex-direction: column; + justify-content: flex-end; + margin-top: -2px; + + &.full { + .battery__charge__body__cell { + &:first-child { + border-radius: 4px 4px 0 0; + } + } + } + + .battery__charge__body__cell { + width: 100%; + height: $battery-height * 0.16; + background-color: transparent; + margin-top: 0.2rem; + + &:first-child { + margin-top: 0; + } + &:last-child { + border-radius: 0 0 4px 4px; + } + } + } + } +} + +.gauge-double { + display: flex; + flex-direction: row; + justify-content: center; + height: 40%; +} + +.gauge-indicator.glue-gauges { + flex: 0 0 auto; + width: 50%; +} +.double-gauges { + display: flex; + width: 50%; +} + +.gauge-label { + display: flex; + align-items: flex-end; + margin-bottom: 5px; + color: $color-gray; + font-size: 0.8rem; + &--left { + margin-right: 0.25rem; + } + &--right { + margin-left: 0.25rem; + } +} + +.power-indicator { + position: absolute; + font-size: 1.4rem; + bottom: 0; + left: 100%; + transform: translate(-50%, -25%); + font-weight: 500; +} \ No newline at end of file diff --git a/venus-html5-app/src/app/KVNRV/components/Battery/Battery.tsx b/venus-html5-app/src/app/KVNRV/components/Battery/Battery.tsx new file mode 100644 index 0000000..25761e2 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Battery/Battery.tsx @@ -0,0 +1,208 @@ +import { Card, SIZE_SHORT } from "../../../components/Card" + +import { BATTERY_STATE } from "../../../utils/constants" +import { useSystemBatteries } from "@victronenergy/mfd-modules" +import { useSendUpdate } from "../../modules" +import NumericValue, { formatNumber } from "../../../components/NumericValue" +import { NotAvailable } from "../NotAvailable" + +import "./Battery.scss" +import { BATTERY_CONF, CRITICAL_MULTIPLIER, STATUS_LEVELS, STATUS_LEVELS_MSG } from "../../utils/constants" +import { normalizePower } from "../../utils/helpers" +import { Footer } from "../../../components/Card/Card" +import { translate, Translate } from "react-i18nify" +import { observer } from "mobx-react" +import { KVNGauge } from "../KVNGauge" + +const batteryStateFormatter = (value: number) => { + switch (value) { + case BATTERY_STATE.CHARGING: + return "charging" + case BATTERY_STATE.DISCHARGING: + return "discharging" + case BATTERY_STATE.IDLE: + return "idle" + default: + return null + } +} + +const batteryTimeToGoFormatter = (timeToGo: number) => { + let secs = timeToGo + if (!isNaN(secs)) { + const days = Math.floor(secs / 86400) + secs = secs - days * 86400 + const hours = Math.floor(secs / 3600) + secs = secs - hours * 3600 + const minutes = Math.floor(secs / 60) + + let time = [] + if (days) time.push(`${days} ${translate("common.days")}`) + if (hours) time.push(`${hours} ${translate("common.hours")}`) + if (minutes) time.push(`${minutes} ${translate("common.minutes")}`) + return time.join(", ") + // we are not interested in seconds, since it's an + // estimate anyways + } else { + return " - " + } +} + +const CELL_NUMBER = 5 +const WARNING_LEVEL = 2 +const ALARM_LEVEL = 1 + +function getClassname(idx: number, batteryLevelBars: number) { + let c = "" + + if (idx < batteryLevelBars) { + c += " success" + } + + if (batteryLevelBars <= ALARM_LEVEL) { + c += " alarm" + } else if (batteryLevelBars <= WARNING_LEVEL) { + c += " warning" + } + + return c +} + +type BatteryProps = { + size: string[] +} + +export const Batteries = observer(({ size }: BatteryProps) => { + const { batteries } = useSystemBatteries() + + const battery = batteries + ? batteries.length > 1 + ? batteries.filter((b) => b.active_battery_service)[0] + : batteries[0] + : undefined + + const power = battery?.power ?? 1 + + const config = { + ...BATTERY_CONF, + MAX: BATTERY_CONF.MAX * (battery?.voltage ? battery.voltage : 1) * CRITICAL_MULTIPLIER, + } + + const normalizedPower = normalizePower(power, config.MAX, -1 * BATTERY_CONF.ZERO_OFFSET!) + + useSendUpdate(normalizedPower, config, "Battery") + + if (batteries) { + if (battery) { + const batteryStateLabel = batteryStateFormatter(battery.state) + const batteryLevelBars = Math.ceil(battery.soc / (100 / CELL_NUMBER)) + + const status = + batteryLevelBars <= ALARM_LEVEL + ? STATUS_LEVELS.ALARM + : batteryLevelBars <= WARNING_LEVEL + ? STATUS_LEVELS.WARNING + : STATUS_LEVELS.SUCCESS + + const footer: Footer = { status: status, message: STATUS_LEVELS_MSG[status], property: "Charge" } + return ( + } size={size} footer={footer}> +
+
+
+
+
+ {batteryLevelBars > 0 && + Array.from(Array(batteryLevelBars).keys()) + .reverse() + .map((idx) => ( +
+ ))} +
+
+
+
+ +
+
+ {batteryStateLabel && ( +
+ +
+ )} +
+ +
+
+ + + + +
+
+ + + + +
+
+
+
+
+
+ +
+
{batteryTimeToGoFormatter(battery.timetogo)}
+
+
+ + + +
+ + <> +
+ {formatNumber({ unit: "W", value: Math.abs(battery.power) })?.toString()} +
+ + + 0} + className="glue-gauges" + percent={battery.power > 0 ? normalizedPower : 0} + from={(-1 * Math.PI) / 2} + to={Math.PI / 2} + parts={BATTERY_CONF.THRESHOLDS} + showText={false} + /> +
+ + + +
+ + ) + } + } + return ( + } size={size}> +
+ +
+ + ) +}) + +export default Batteries diff --git a/venus-html5-app/src/app/KVNRV/components/Battery/index.js b/venus-html5-app/src/app/KVNRV/components/Battery/index.js new file mode 100644 index 0000000..5e7edae --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Battery/index.js @@ -0,0 +1 @@ +export { default } from "./Battery" diff --git a/venus-html5-app/src/app/KVNRV/components/DcLoads/DcLoads.scss b/venus-html5-app/src/app/KVNRV/components/DcLoads/DcLoads.scss new file mode 100644 index 0000000..afb4031 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/DcLoads/DcLoads.scss @@ -0,0 +1,2 @@ +@import "../../css/variables"; + diff --git a/venus-html5-app/src/app/KVNRV/components/DcLoads/DcLoads.tsx b/venus-html5-app/src/app/KVNRV/components/DcLoads/DcLoads.tsx new file mode 100644 index 0000000..8236a6f --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/DcLoads/DcLoads.tsx @@ -0,0 +1,37 @@ +import { Card, SIZE_NARROW, SIZE_SHORT } from "../../../components/Card" +import { useDcLoads } from "@victronenergy/mfd-modules" +import { useSendUpdate } from "../../modules" +import { CRITICAL_MULTIPLIER, DC_CONF } from "../../utils/constants" + +import "./DcLoads.scss" +import NumericValue from "../../../components/NumericValue" +import { normalizePower } from "../../utils/helpers" +import { Translate } from "react-i18nify" +import { observer } from "mobx-react" +import { KVNGauge } from "../KVNGauge" + +export const DcLoads = observer(() => { + const { voltage, current, power } = useDcLoads() + + const powerMax = (voltage ?? 1) * 60 * CRITICAL_MULTIPLIER + const normalizedPower = normalizePower(power ?? 0, powerMax) + useSendUpdate(normalizedPower, DC_CONF, "DC Loads") + + return ( + } size={[SIZE_SHORT, SIZE_NARROW]}> +
+ +
+
+ +
+
+ +
+
+
+ + ) +}) + +export default DcLoads diff --git a/venus-html5-app/src/app/KVNRV/components/DcLoads/index.js b/venus-html5-app/src/app/KVNRV/components/DcLoads/index.js new file mode 100644 index 0000000..d341c6d --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/DcLoads/index.js @@ -0,0 +1 @@ +export { default } from "./DcLoads" diff --git a/venus-html5-app/src/app/KVNRV/components/Header/Header.scss b/venus-html5-app/src/app/KVNRV/components/Header/Header.scss new file mode 100644 index 0000000..b3f6b99 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Header/Header.scss @@ -0,0 +1,211 @@ +@import "../../css/variables"; + +.header { + display: flex; + flex-direction: row; + justify-content: space-between; + padding: 1rem; + align-items: center; + background-color: $header-bg-color; + color: $color-white; + + .header__left { + flex: 1; + display: flex; + flex-direction: row; + align-items: center; + + + .header__logo { + .header__logo__image { + vertical-align: middle; + margin-right: 0.5rem; + width: 3rem; + } + + .header__logo__text { + font-family: Tommy, sans-serif; + font-size: 2rem; + font-weight: 900; + vertical-align: middle; + + @media all and (max-width: 650px) { + display: none; + } + } + } + + .header__info { + border-left: white solid 2px; + padding-left: 1rem; + font-size: 1rem; + margin-right: 0.8rem; + margin-left: 0.8rem; + + @media #{$regular} { + display: none; + } + @media #{$minimum} { + display: none; + } + } + } + + .header__buttons { + flex: 1; + display: flex; + flex-direction: row; + height: 3rem; + justify-content: flex-end; + + div { + margin-right: 1rem; + } + .header__buttons__icon { + max-height: 1.4rem; + } + + .header__buttons__remote-console { + font-size: 1.4rem; + background-color: $header-button-color; + color: $color-white; + padding: 0.2rem 0.8rem; + border: none; + border-radius: 5px; + margin-right: 0.8rem; + + @media #{$hide-remote-console-breakpoint} { + display: none; + } + + .header__buttons__icon { + vertical-align: middle; + } + .header__buttons__text { + margin-left: 0.5rem; + vertical-align: middle; + font-size: 0.9rem; + + @media all and (max-width: 690px) { + display: none; + } + } + } + + .header__buttons__logout { + display: flex; + align-items: center; + justify-content: center; + justify-self: flex-start; + background-color: $header-button-color; + color: $color-white; + padding: 0.2rem 1.2rem 0.2rem 0.8rem; + margin-right: auto; + border: none; + border-radius: 5px; + + .header__buttons__icon { + width: 2rem; + vertical-align: middle; + } + } + + $theme-slider-width: 6rem; + $theme-slider-height: 1.75rem; + $theme-slider-circle-size: $theme-slider-height * 1.5; + + .header__buttons__remote-connection { + display: flex; + flex-direction: row; + align-self: center; + + button { + border-radius: 2rem; + background-color: $header-button-color; + color: $color-white; + height: $theme-slider-height; + align-self: center; + + display: block; + z-index: 1; + font-size: 1.2rem; + + &.active { + border-radius: 2rem; + background-color: $color-white; + color: $header-bg-color; + height: 2.2rem; + + z-index: 2; + } + } + + .remote { + padding-left: 0.8rem; + padding-right: 1.2rem; + margin-right: -1rem; + + &.active { + padding-right: 0.5rem; + margin-right: 0; + } + } + + .local { + padding-right: 0.8rem; + padding-left: 1.2rem; + margin-left: -1rem; + + &.active { + padding-left: 0.5rem; + margin-left: 0; + } + } + } + + .header__buttons__darkmode { + align-self: center; + + .header__buttons__darkmode__switch { + position: relative; + display: inline-block; + height: $theme-slider-height; + width: $theme-slider-width; + vertical-align: middle; + } + + #header__buttons__darkmode__input { + display: none; + } + + .header__buttons__darkmode__slider { + background-color: $header-button-color; + bottom: 0; + cursor: pointer; + left: 0; + position: absolute; + right: 0; + top: 0; + transition: .4s; + border-radius: 1rem; + + .header__buttons__darkmode__slider__img { + bottom: -1 * ($theme-slider-height * 0.5) * 0.5; + left: 0; + position: absolute; + transition: .1s; + height: $theme-slider-circle-size; + width: $theme-slider-circle-size; + border-radius: 50%; + background: $color-white; + padding: 0.4rem; + } + } + + + #header__buttons__darkmode__input:checked + .header__buttons__darkmode__slider > .header__buttons__darkmode__slider__img { + transform: translateX($theme-slider-width - $theme-slider-circle-size); + } + } + } +} diff --git a/venus-html5-app/src/app/KVNRV/components/Header/Header.tsx b/venus-html5-app/src/app/KVNRV/components/Header/Header.tsx new file mode 100644 index 0000000..22dc68f --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Header/Header.tsx @@ -0,0 +1,118 @@ +import { useEffect, useRef } from "react" +import { useApp, useTheme, useVrmStore } from "@victronenergy/mfd-modules" +import { VIEWS } from "../../utils/constants" + +import "./Header.scss" + +import KVNRVLogo from "../../images/KVNRV-Logo.svg" +import RemoteConsoleIcon from "../../images/RemoteConsoleIcon.svg" +import LogOutIcon from "../../images/LogOut.svg" +import LightThemeIcon from "../../images/LightThemeIcon.svg" +import DarkThemeIcon from "../../images/DarkThemeIcon.svg" +import { Translate } from "react-i18nify" +import { observer } from "mobx-react" +import { ModalVersionInfo } from "../ModalVersionInfo" + +export const Header = observer(() => { + const { darkMode, themeStore } = useTheme() + const appStore = useApp() + const remote = appStore.remote + const vrmStore = useVrmStore() + const { loggedIn, username, siteId } = vrmStore + const modalVersionInfoRef = useRef() + + const handleRemoteSwitch = () => { + window.location.replace(remote ? `http://venus.local/app` : `https://kvnrv-9ca32.web.app/app`) + appStore.toggleRemote() + } + + useEffect(() => { + if (remote && (!loggedIn || !siteId)) { + appStore.setPage(VIEWS.LOGIN) + } else { + appStore.setPage(VIEWS.METRICS) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [remote, loggedIn, siteId]) + + return ( + <> +
+
+
{ + modalVersionInfoRef.current.open() + }} + > + {"KVNRV + KVNRV +
+ +
+ {loggedIn && ( + <> +
{username && }
+
{username ?? }
+ + )} +
+ +
+ <> + {loggedIn && ( + + )} + +
+
+ +
+
handleRemoteSwitch()}> + + +
+
+ +
+ + {!remote && ( + + )} +
+
+ + + ) +}) + +export default Header diff --git a/venus-html5-app/src/app/KVNRV/components/Header/index.js b/venus-html5-app/src/app/KVNRV/components/Header/index.js new file mode 100644 index 0000000..e635b53 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Header/index.js @@ -0,0 +1 @@ +export { Header, default } from "./Header" diff --git a/venus-html5-app/src/app/KVNRV/components/Installations/Installations.scss b/venus-html5-app/src/app/KVNRV/components/Installations/Installations.scss new file mode 100644 index 0000000..7c47d8f --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Installations/Installations.scss @@ -0,0 +1,38 @@ +@import "../../css/variables"; + +.installations { + display: flex; + flex-direction: column; + justify-content: center; + margin-top: 1em; +} + +.installations__title { + text-align: center; + margin: 1em auto; +} + +.installations__list { + display: flex; + flex-direction: column; + max-height: 40vh; + width: 24em; + max-width: 90vw; + overflow-y: scroll; +} + +.installations__list__item { + display: flex; + flex-direction: row; + padding: 12px 8px; + border: none; + border-radius: 0; + background-color: $card-bg-color; + color: $text-color-main; + font-size: 1em; + + &:hover { + color: $color-white; + background-color: $color-login-blue; + } +} diff --git a/venus-html5-app/src/app/KVNRV/components/Installations/Installations.tsx b/venus-html5-app/src/app/KVNRV/components/Installations/Installations.tsx new file mode 100644 index 0000000..7df512f --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Installations/Installations.tsx @@ -0,0 +1,35 @@ +import { useAppStore, useVrmStore } from "@victronenergy/mfd-modules" +import "./Installations.scss" +import { VIEWS } from "../../utils/constants" +import { Translate } from "react-i18nify" +import { observer } from "mobx-react" + +export const Installations = observer(() => { + const vrmStore = useVrmStore() + const { installations } = vrmStore + const appStore = useAppStore() + + const selectInstallation = (id: number) => { + vrmStore.setActiveInstallation(id) + appStore.setPage(VIEWS.METRICS) + } + + return ( +
+
+ +
+
+ {installations?.map((installation) => ( + + ))} +
+
+ ) +}) diff --git a/venus-html5-app/src/app/KVNRV/components/Installations/NoInstallations.tsx b/venus-html5-app/src/app/KVNRV/components/Installations/NoInstallations.tsx new file mode 100644 index 0000000..925d850 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Installations/NoInstallations.tsx @@ -0,0 +1,24 @@ +import { useVrmStore } from "@victronenergy/mfd-modules" +import { Translate } from "react-i18nify" + +export const NoInstallations = () => { + const vrmStore = useVrmStore() + + return ( +
+
+ +
+ + +
+ ) +} diff --git a/venus-html5-app/src/app/KVNRV/components/Installations/index.ts b/venus-html5-app/src/app/KVNRV/components/Installations/index.ts new file mode 100644 index 0000000..3a3d657 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Installations/index.ts @@ -0,0 +1,2 @@ +export * from "./Installations" +export * from "./NoInstallations" diff --git a/venus-html5-app/src/app/KVNRV/components/KVNGauge/GaugeIndicator.scss b/venus-html5-app/src/app/KVNRV/components/KVNGauge/GaugeIndicator.scss new file mode 100644 index 0000000..61e2d27 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/KVNGauge/GaugeIndicator.scss @@ -0,0 +1,19 @@ + +@import "../../css/variables"; + +$size-multiplier: 1.5; +$small-chart-width-factor: 0.78; +$small-chart-height-factor: 0.6; + +.gauge-indicator { + position: relative; + width: 100%; + height: 100%; + align-self: center; +} + + +.canvas { + width: 100%; + height: 100%; +} \ No newline at end of file diff --git a/venus-html5-app/src/app/KVNRV/components/KVNGauge/index.tsx b/venus-html5-app/src/app/KVNRV/components/KVNGauge/index.tsx new file mode 100644 index 0000000..8580f0a --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/KVNGauge/index.tsx @@ -0,0 +1,216 @@ +import React from "react" +import { sum } from "app/KVNRV/utils/helpers" +import { useContainerColors } from "app/KVNRV/utils/hooks" +import { Chart } from "chart.js" +import { useCallback, useEffect, useMemo, useRef, ReactNode } from "react" +import "./GaugeIndicator.scss" +import { TextPlugin } from "./plugins/TextPlugin" +import { debounce } from "lodash-es" +import { observer } from "mobx-react" + +const defaultOptions = { + maintainAspectRatio: false, + responsive: true, + rotation: 0, + circumference: 180, + legend: { + display: false, + }, + tooltips: { + enabled: false, + }, + cutoutPercentage: 60, +} + +type GaugeIndicatorProps = { + value?: number + unit?: string + percent: number + parts: Array + gauge?: boolean + from?: number + to?: number + children?: ReactNode + inverse?: boolean + showText?: boolean + className?: string + showNeedle?: boolean +} + +const INDICATOR_WIDTH = 0.015 + +export const KVNGauge = observer( + ({ + value, + percent, + parts, + className = "", + unit, + children, + inverse = false, + showText = true, + from = -Math.PI, + to = Math.PI, + showNeedle = true, + }: GaugeIndicatorProps) => { + const canvasEl = useRef() as React.MutableRefObject + + const chartRef = useRef() + const colors = useContainerColors() + + // colors used for threshold circle + const orderedColors = useMemo(() => { + const { colorGreen, colorOrange, colorRed } = colors + const clrs = [colorGreen, colorOrange, colorRed] + return !inverse ? clrs : clrs.reverse() + }, [colors, inverse]) + + // color used for the left part of the needle + const indicatorColor = useMemo(() => { + const indexOfPart = parts.findIndex((_, idx, arr) => sum(arr.slice(0, idx + 1)) >= percent) + const usedColor = orderedColors[indexOfPart] || colors.colorGray + return usedColor + }, [colors.colorGray, orderedColors, parts, percent]) + + // colors of inner circle + const indicatorColors = useMemo(() => { + const clrs = [indicatorColor, colors.textColor /* needle color */, colors.colorGray /* empty zone color */] + return !inverse ? clrs : clrs.reverse() + }, [indicatorColor, colors.textColor, colors.colorGray, inverse]) + + const indicatorPoints = useMemo(() => { + const indicatorWidth = showNeedle ? INDICATOR_WIDTH : 0 + return [percent - indicatorWidth / 2, indicatorWidth, 1 - percent - indicatorWidth / 2] + }, [percent, showNeedle]) + + // function used to get first data on first render + const getIndicatorData = useCallback(() => { + return { + data: indicatorPoints, + weight: 4, + spacing: 10, + + // add white border to the needle + borderColor: ["transparent", colors.textColor, "transparent"], + hoverBorderColor: ["transparent", colors.textColor, "transparent"], + borderWidth: [0, showNeedle ? 1 : 0, 0], + + backgroundColor: indicatorColors, + hoverBackgroundColor: indicatorColors, + } + }, [indicatorPoints, colors.textColor, showNeedle, indicatorColors]) + + const createChart = useCallback(() => { + if (!canvasEl.current) { + return + } + + const chartCanvas = canvasEl.current.getContext("2d") + + if (!chartCanvas) { + return + } + + chartRef.current = new Chart(chartCanvas, { + type: "doughnut", + plugins: [showText ? TextPlugin() : {}], + options: { + ...defaultOptions, + rotation: from, + circumference: to, + }, + data: { + datasets: [ + { + data: [...parts], + weight: 1, + backgroundColor: orderedColors, + hoverBackgroundColor: orderedColors, + borderColor: "transparent", + }, + { + data: [1], + weight: 1, + borderColor: "transparent", + backgroundColor: "transparent", + hoverBackgroundColor: "transparent", + }, + getIndicatorData(), + ], + }, + }) + // inject options for text plugin + //@ts-ignore + chartRef.current.options.textPlugin = { textColor: colors.textColor, value, unit } + }, [colors.textColor, from, getIndicatorData, orderedColors, parts, showText, to, unit, value]) + + // create chart entity on first mount + useEffect(() => { + if (!canvasEl.current) { + return + } + + // create chart only if it does not exist + if (!chartRef.current) { + createChart() + } + }, [createChart]) + + // update threshold dataset + useEffect(() => { + if (!chartRef.current || !chartRef.current.data.datasets) { + return + } + + chartRef.current.data.datasets[0].backgroundColor = orderedColors + chartRef.current.data.datasets[0].hoverBackgroundColor = orderedColors + + chartRef.current.data.datasets[0].data = parts + chartRef.current.update() + }, [orderedColors, parts]) + + // update indicator dataset + useEffect(() => { + if (!chartRef.current || !chartRef.current.data.datasets) { + return + } + + chartRef.current.data.datasets[2].data = indicatorPoints + chartRef.current.data.datasets[2].backgroundColor = indicatorColors + chartRef.current.data.datasets[2].hoverBackgroundColor = indicatorColors + chartRef.current.data.datasets[2].borderColor = ["transparent", colors.textColor, "transparent"] + chartRef.current.data.datasets[2].hoverBorderColor = ["transparent", colors.textColor, "transparent"] + + // update options for text plugin + //@ts-ignore + chartRef.current.options.textPlugin = { textColor: colors.textColor, value, unit } + + chartRef.current.update() + }, [colors.textColor, indicatorColors, indicatorPoints, unit, value]) + + const onRespawn = function () { + if (chartRef.current) { + chartRef.current.destroy() + createChart() + } + } + + // Chart js was not updating it's height and width on resize + // below we destroy and create it again in order for it to resize accordingly + useEffect(() => { + const debouncedResize = debounce(onRespawn, 500) + window.addEventListener("resize", debouncedResize) + return () => window.removeEventListener("resize", debouncedResize) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [createChart]) + + return ( +
+ {children} +
+ +
+
+ ) + }, +) diff --git a/venus-html5-app/src/app/KVNRV/components/KVNGauge/plugins/TextPlugin.tsx b/venus-html5-app/src/app/KVNRV/components/KVNGauge/plugins/TextPlugin.tsx new file mode 100644 index 0000000..7da117e --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/KVNGauge/plugins/TextPlugin.tsx @@ -0,0 +1,32 @@ +import { formatNumber } from "app/components/NumericValue" + +export const TextPlugin = () => ({ + // eslint-disable-next-line no-undef + beforeDraw: (chart: Chart) => { + let width = chart.width, + height = chart.height, + ctx = chart.ctx + if (!ctx || !height || !width) { + return + } + //@ts-ignore + const textPluginOptions = chart.options.textPlugin + ctx.restore() + let fontSize = Math.min(width / 120, 1.4).toFixed(2) + // For some reason in Safari rem results in tiny letters, so use em + ctx.font = fontSize + "em sans-serif" + ctx.textBaseline = "middle" + ctx.fillStyle = textPluginOptions.textColor + + let text = + formatNumber({ + value: textPluginOptions.value, + unit: textPluginOptions.unit, + })?.toString() ?? "", + textX = Math.round((width - ctx.measureText(text).width) / 2), + textY = height / 2 + height * 0.25 + + ctx.fillText(text, textX, textY) + ctx.save() + }, +}) diff --git a/venus-html5-app/src/app/KVNRV/components/ModalVersionInfo/ModalVersionInfo.scss b/venus-html5-app/src/app/KVNRV/components/ModalVersionInfo/ModalVersionInfo.scss new file mode 100644 index 0000000..c12d234 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/ModalVersionInfo/ModalVersionInfo.scss @@ -0,0 +1,47 @@ +@import "../../css/variables"; + +.modal-ver-container { + display: flex; + width: 100%; + margin-top: 1rem; + .left-info { + width: 100%; + justify-content: center; + align-items: center; + display: flex; + flex-direction: column; + + &-content { + .version-logo { + display: flex; + align-items: center; + span { + margin-left: 0.6rem; + font-size: 1.3rem; + } + } + .version-item { + margin-top: 0.5rem; + font-size: 0.8rem; + color: $color-gray; + } + } + } + + .right-info { + width: 100%; + display: flex; + .first-column, .second-column { + display: flex; + flex-direction: column; + & p { + margin-top: 0; + margin-bottom: 0.7rem; + } + } + .second-column { + margin-left: 2rem; + color: $color-gray; + } + } +} diff --git a/venus-html5-app/src/app/KVNRV/components/ModalVersionInfo/ModalVersionInfo.tsx b/venus-html5-app/src/app/KVNRV/components/ModalVersionInfo/ModalVersionInfo.tsx new file mode 100644 index 0000000..95ea91d --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/ModalVersionInfo/ModalVersionInfo.tsx @@ -0,0 +1,74 @@ +import Modal from "app/components/Modal" +import { forwardRef, useImperativeHandle, useState } from "react" +import { translate, Translate } from "react-i18nify" +import "./ModalVersionInfo.scss" +import KVNRVLogo from "../../images/KVNRV-Logo.svg" +import { SIZE_EXTRA_WIDE } from "app/components/Card" +import { useVrmStore, useAppStore } from "@victronenergy/mfd-modules" +import { BUILD_TIMESTAMP } from "app/utils/constants" +import { observer } from "mobx-react" +import packageInfo from "../../../../../package.json" + +export const ModalVersionInfo = observer( + // eslint-disable-next-line react/display-name + forwardRef((_, ref) => { + const [isOpen, setOpen] = useState(false) + const { portalId = "-", siteId = "-" } = useVrmStore() + const { humanReadableFirmwareVersion } = useAppStore() + + useImperativeHandle(ref, () => ({ + open: () => setOpen(true), + })) + + return ( + <> + {isOpen && ( + setOpen(false)} + title={translate("header.versionInfo")} + > +
+
+
+
+ {"KVNRV + KVNRV +
+
+ +
+
+
+
+
+

+ +

+

+ +

+

+ +

+

+ +

+
+
+

{BUILD_TIMESTAMP}

+

{humanReadableFirmwareVersion}

+

{portalId}

+

{siteId}

+
+
+
+ + )} + + ) + }), +) diff --git a/venus-html5-app/src/app/KVNRV/components/ModalVersionInfo/index.js b/venus-html5-app/src/app/KVNRV/components/ModalVersionInfo/index.js new file mode 100644 index 0000000..ac64ca9 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/ModalVersionInfo/index.js @@ -0,0 +1 @@ +export { ModalVersionInfo } from "./ModalVersionInfo" diff --git a/venus-html5-app/src/app/KVNRV/components/NotAvailable/NotAvailable.scss b/venus-html5-app/src/app/KVNRV/components/NotAvailable/NotAvailable.scss new file mode 100644 index 0000000..5795756 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/NotAvailable/NotAvailable.scss @@ -0,0 +1,10 @@ +@import "../../css/variables"; + +.not-available { + height: $donut-height-big; + display: flex; + align-self: center; + align-content: center; + align-items: center; + color: $color-gray; +} diff --git a/venus-html5-app/src/app/KVNRV/components/NotAvailable/NotAvailable.tsx b/venus-html5-app/src/app/KVNRV/components/NotAvailable/NotAvailable.tsx new file mode 100644 index 0000000..7641819 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/NotAvailable/NotAvailable.tsx @@ -0,0 +1,8 @@ +import { Translate } from "react-i18nify" +import "./NotAvailable.scss" + +export const NotAvailable = () => ( +
+ +
+) diff --git a/venus-html5-app/src/app/KVNRV/components/NotAvailable/index.js b/venus-html5-app/src/app/KVNRV/components/NotAvailable/index.js new file mode 100644 index 0000000..3167f83 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/NotAvailable/index.js @@ -0,0 +1 @@ +export { NotAvailable } from "./NotAvailable" diff --git a/venus-html5-app/src/app/KVNRV/components/Paginator/Paginator.scss b/venus-html5-app/src/app/KVNRV/components/Paginator/Paginator.scss new file mode 100644 index 0000000..38aeed6 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Paginator/Paginator.scss @@ -0,0 +1,25 @@ +@import "../../css/variables"; + +.paginator { + display: flex; + flex-direction: row; + justify-content: center; + padding-bottom: 6rem; + .paginator__icon { + width: 2rem; + height: 2rem; + padding: 0.3rem 0.5rem ; + margin: 0.6rem 1rem; + background-color: $paginator-arrow-color; + border-radius: 5px; + cursor: pointer; + } + + .paginator__dot { + width: 0.8rem; + height: 0.8rem; + border-radius: 50%; + background-color: $color-lightgray; + margin: auto 0.4rem; + } +} diff --git a/venus-html5-app/src/app/KVNRV/components/Paginator/Paginator.tsx b/venus-html5-app/src/app/KVNRV/components/Paginator/Paginator.tsx new file mode 100644 index 0000000..af775af --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Paginator/Paginator.tsx @@ -0,0 +1,43 @@ +import React from "react" + +import LIcon from "../../../../images/icons/L.svg" +import RIcon from "../../../../images/icons/R.svg" +import "./Paginator.scss" + +type PaginatorProps = { + pages: number + currentPage: number + setCurrentPage: Function +} + +export const Paginator = ({ pages, currentPage, setCurrentPage }: PaginatorProps) => { + const updatePage = (pageNumber: number) => { + if (pageNumber >= 0 && pageNumber < pages) { + setCurrentPage(pageNumber) + } + } + + return ( +
+ {"Paginator updatePage(currentPage - 1)} + /> + + {Array.from(Array(pages).keys()).map((i) => ( +
+ ))} + + {"Paginator updatePage(currentPage + 1)} + /> +
+ ) +} + +export default Paginator diff --git a/venus-html5-app/src/app/KVNRV/components/Paginator/index.js b/venus-html5-app/src/app/KVNRV/components/Paginator/index.js new file mode 100644 index 0000000..d4e719a --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Paginator/index.js @@ -0,0 +1 @@ +export { default } from "./Paginator" diff --git a/venus-html5-app/src/app/KVNRV/components/ProgressIndicator/ProgressIndicator.scss b/venus-html5-app/src/app/KVNRV/components/ProgressIndicator/ProgressIndicator.scss new file mode 100644 index 0000000..51d51fe --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/ProgressIndicator/ProgressIndicator.scss @@ -0,0 +1,14 @@ +@import "../../css/variables"; + +.progress-indicator { + border-radius: $progress-radius; + width: 100%; + height: $progress-height; + background-color: $color-lightgray; + + + .progress-indicator__bar { + border-radius: $progress-radius; + height: $progress-height; + } +} diff --git a/venus-html5-app/src/app/KVNRV/components/ProgressIndicator/ProgressIndicator.tsx b/venus-html5-app/src/app/KVNRV/components/ProgressIndicator/ProgressIndicator.tsx new file mode 100644 index 0000000..99fb094 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/ProgressIndicator/ProgressIndicator.tsx @@ -0,0 +1,25 @@ +import React from "react" +import "./ProgressIndicator.scss" + +type ProgressIndicatorProps = { + percent: number + level: string +} + +export const ProgressIndicator = (props: ProgressIndicatorProps) => { + let percent = props.percent * 100 + if (percent < 5) { + percent = 0 + } else if (percent < 10) { + percent = 10 + } + const styles = { width: percent + "%" } + + return ( +
+
+
+ ) +} + +export default ProgressIndicator diff --git a/venus-html5-app/src/app/KVNRV/components/ProgressIndicator/index.js b/venus-html5-app/src/app/KVNRV/components/ProgressIndicator/index.js new file mode 100644 index 0000000..6b64d02 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/ProgressIndicator/index.js @@ -0,0 +1 @@ +export { default } from "./ProgressIndicator" diff --git a/venus-html5-app/src/app/KVNRV/components/PvCharger/PvCharger.tsx b/venus-html5-app/src/app/KVNRV/components/PvCharger/PvCharger.tsx new file mode 100644 index 0000000..bc43bec --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/PvCharger/PvCharger.tsx @@ -0,0 +1,35 @@ +import React from "react" + +import { PV_CONF } from "../../utils/constants" +import { usePvCharger } from "@victronenergy/mfd-modules" +import { useSendUpdate } from "../../modules" +import { Card, SIZE_NARROW, SIZE_SHORT } from "../../../components/Card" +import NumericValue from "../../../components/NumericValue" +import { normalizePower } from "../../utils/helpers" +import { Translate } from "react-i18nify" +import { observer } from "mobx-react" +import { KVNGauge } from "../KVNGauge" + +export const PvCharger = observer(() => { + const { current, power } = usePvCharger() + + const normalizedPower = normalizePower(power ?? 0, PV_CONF.MAX) + useSendUpdate(normalizedPower, PV_CONF, "PV Charger") + + return ( +
+ } size={[SIZE_SHORT, SIZE_NARROW]}> +
+ +
+
+ +
+
+
+ +
+ ) +}) + +export default PvCharger diff --git a/venus-html5-app/src/app/KVNRV/components/PvCharger/index.js b/venus-html5-app/src/app/KVNRV/components/PvCharger/index.js new file mode 100644 index 0000000..1721380 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/PvCharger/index.js @@ -0,0 +1 @@ +export { default } from "./PvCharger" diff --git a/venus-html5-app/src/app/KVNRV/components/ShorePower/ShorePower.scss b/venus-html5-app/src/app/KVNRV/components/ShorePower/ShorePower.scss new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/ShorePower/ShorePower.scss @@ -0,0 +1 @@ + diff --git a/venus-html5-app/src/app/KVNRV/components/ShorePower/ShorePower.tsx b/venus-html5-app/src/app/KVNRV/components/ShorePower/ShorePower.tsx new file mode 100644 index 0000000..264def1 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/ShorePower/ShorePower.tsx @@ -0,0 +1,47 @@ +import { Card, SIZE_NARROW, SIZE_SHORT } from "../../../components/Card" + +import "./ShorePower.scss" +import NumericValue from "../../../components/NumericValue" +import { useActiveInValues } from "@victronenergy/mfd-modules" +import { useSendUpdate } from "../../modules" +import { SHORE_POWER_CONF } from "../../utils/constants" +import { normalizePower } from "../../utils/helpers" +import { NotAvailable } from "../NotAvailable" +import { Translate } from "react-i18nify" +import { observer } from "mobx-react" +import { KVNGauge } from "../KVNGauge" + +export const ShorePower = observer(() => { + const { current, frequency, voltage, power } = useActiveInValues() + const normalizedPower = normalizePower(power && power[0] ? power[0] : 0, SHORE_POWER_CONF.MAX) + useSendUpdate(normalizedPower, SHORE_POWER_CONF, "Shore Power") + + return ( +
+ } size={[SIZE_SHORT, SIZE_NARROW]}> +
+ {power ? ( + <> + + + ) : ( + + )} +
+
+ +
+
+ +
+
+ +
+
+
+ +
+ ) +}) + +export default ShorePower diff --git a/venus-html5-app/src/app/KVNRV/components/ShorePower/index.js b/venus-html5-app/src/app/KVNRV/components/ShorePower/index.js new file mode 100644 index 0000000..4793e69 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/ShorePower/index.js @@ -0,0 +1 @@ +export { default } from "./ShorePower" diff --git a/venus-html5-app/src/app/KVNRV/components/Status/Status.scss b/venus-html5-app/src/app/KVNRV/components/Status/Status.scss new file mode 100644 index 0000000..9fbece0 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Status/Status.scss @@ -0,0 +1,61 @@ +@import "../../css/variables"; + +.metrics__status { + .title { + font-size: $indicator-main-font-value-big; + } + + .subheading { + display: flex; + font-size: $indicator-main-font-name-small; + color: $color-gray; + justify-content: space-between; + margin-top: 0.6rem; + } + + .status-updates { + margin-top: 1rem; + height: 12rem; + overflow-y: auto; + + .status-update { + margin-top: 1rem; + border-radius: 5px; + padding: 0.3rem; + color: white; + font-size: 0.9rem; + justify-content: space-between; + align-items: center; + + &.warning { + .status-update__icon { + background-color: $color-yellow-fade; + } + } + &.alarm { + .status-update__icon { + background-color: $color-red-fade; + } + } + + .status-update__message { + margin-left: 0.2rem; + } + + .status-update__icon { + margin-right: 0.2rem; + border-radius: 5px; + width: 1.3rem; + height: 1.3rem; + align-self: center; + + img { + width: 1rem; + height: 1rem; + margin-top: auto; + margin-bottom: auto; + } + } + } + } +} diff --git a/venus-html5-app/src/app/KVNRV/components/Status/Status.tsx b/venus-html5-app/src/app/KVNRV/components/Status/Status.tsx new file mode 100644 index 0000000..47f1020 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Status/Status.tsx @@ -0,0 +1,111 @@ +import React from "react" + +import { Card } from "../../../components/Card" +import IconWarning from "../../../images/IconWarning.svg" +import "./Status.scss" +import { STATUS_LEVELS } from "../../utils/constants" +import { useSystemState } from "@victronenergy/mfd-modules" +import { + BatteryAlarmsState, + StatusUpdate, + useBatteryAlarms, + useStatus, + useVebusAlarms, + VebusAlarmsState, +} from "../../modules" +import { Translate } from "react-i18nify" +import { observer } from "mobx-react" + +const SYSTEM_STATE_MAP = { + 0: "Off", + 1: "Low power", + 2: "VE.Bus Fault condition", + 3: "Bulk charging", + 4: "Absorption charging", + 5: "Float charging", + 6: "Storage mode", + 7: "Equalisation charging", + 8: "Passthru", + 9: "Inverting", + 10: "Assisting", + 256: "Discharging", + 257: "Sustain", +} + +const keyToString = (key: string) => { + return key + .split(/(?=[A-Z])/) + .map((s) => s.toLowerCase()) + .join(" ") +} + +const alarmsToUpdate = (alarms: BatteryAlarmsState | VebusAlarmsState, part?: string) => { + let updates: StatusUpdate[] = [] + Object.keys(alarms).forEach((key) => { + if (alarms[key as keyof typeof alarms] > 0) { + updates.push({ + part: part ? `Venus (${part})` : "Venus", + message: keyToString(key), + level: alarms[key as keyof typeof alarms] === 1 ? STATUS_LEVELS.WARNING : STATUS_LEVELS.ALARM, + } as StatusUpdate) + } + }) + return updates +} + +type StatusProps = { + size: string[] +} + +export const Status = observer(({ size }: StatusProps) => { + const { statuses } = useStatus() + const { systemState } = useSystemState() + + const batteryAlarms = useBatteryAlarms() + const vebusAlarms = useVebusAlarms() + + let notifications: StatusUpdate[] = [] + + notifications = notifications.concat(alarmsToUpdate(batteryAlarms, "Battery")) + notifications = notifications.concat(alarmsToUpdate(vebusAlarms)) + notifications = notifications.concat(statuses?.slice() ?? []) + const status = SYSTEM_STATE_MAP[systemState?.toString() as unknown as keyof typeof SYSTEM_STATE_MAP] + + return ( +
+ } size={size}> +
Penny's House
+
+ +
+
+
Geo Pro Travel Trailer
+
Model: 2019
+
+ +
+ {notifications.map((update: StatusUpdate) => ( +
+ +
+ {"Status +
+ + + {!update.part.toLowerCase().includes("venus") ? ( + + ) : ( + update.part + )} + : {update.message} + + +
+ ))} +
+ +
+ ) +}) + +export default Status diff --git a/venus-html5-app/src/app/KVNRV/components/Status/index.js b/venus-html5-app/src/app/KVNRV/components/Status/index.js new file mode 100644 index 0000000..af0d9e8 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Status/index.js @@ -0,0 +1 @@ +export { default } from "./Status" diff --git a/venus-html5-app/src/app/KVNRV/components/Tanks/SmallTank.scss b/venus-html5-app/src/app/KVNRV/components/Tanks/SmallTank.scss new file mode 100644 index 0000000..a33b919 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Tanks/SmallTank.scss @@ -0,0 +1,9 @@ +@import "../../css/variables"; + +.small-tank { + display: flex; + flex-direction: column; + justify-content: space-between; + height: 80%; + margin-top: 1rem; +} diff --git a/venus-html5-app/src/app/KVNRV/components/Tanks/SmallTank.tsx b/venus-html5-app/src/app/KVNRV/components/Tanks/SmallTank.tsx new file mode 100644 index 0000000..9a5a62a --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Tanks/SmallTank.tsx @@ -0,0 +1,85 @@ +import React from "react" + +import { Card, SIZE_NARROW, SIZE_SHORT } from "../../../components/Card" +import { NotAvailable } from "../NotAvailable" +import NumericValue, { formatNumber } from "../../../components/NumericValue" +import ProgressIndicator from "../ProgressIndicator" + +import "./SmallTank.scss" +import { useTank } from "@victronenergy/mfd-modules" +import { TankProps } from "./index" +import { useSendUpdate } from "../../modules" +import { + VOLUME_UNITS, + VolumeUnit, + VolumeUnits, + FLUID_TYPES, + REVERSE_CONFIG_FLUID_TYPES, + TANKS_CONF, +} from "../../utils/constants" +import { Translate } from "react-i18nify" +import { observer } from "mobx-react" + +export const fluidTypeFormatter = (fluidType: number) => { + switch (fluidType) { + case FLUID_TYPES.FUEL: + return "Fuel" + case FLUID_TYPES.FRESH_WATER: + return "Fresh water" + case FLUID_TYPES.WASTE_WATER: + return "Waste water" + case FLUID_TYPES.LIVE_WELL: + return "Live well" + case FLUID_TYPES.OIL: + return "Oil" + case FLUID_TYPES.BLACK_WATER: + return "Black water" + default: + return "Tank sensor" + } +} + +export const SmallTank = observer(({ tankId }: TankProps) => { + const tank = useTank(tankId) + const hasReverseConfig = REVERSE_CONFIG_FLUID_TYPES.includes(+tank.fluidType) + + const footer = useSendUpdate( + !hasReverseConfig ? 1 - tank.level / 100 : tank.level / 100, + hasReverseConfig ? TANKS_CONF.REVERSE_TANK : TANKS_CONF.STANDART_TANK, + fluidTypeFormatter(tank.fluidType), + ) + + const unit: VolumeUnit = + tank?.unit && Object.keys(VOLUME_UNITS).includes(tank.unit.toString()) + ? VOLUME_UNITS[tank.unit.toString() as keyof VolumeUnits] + : VOLUME_UNITS.default + + return ( +
+ } + size={[SIZE_SHORT, SIZE_NARROW]} + footer={footer} + > +
+ {tank ? ( +
+
+ + + + {formatNumber({ value: tank.remaining * unit.factor, unit: unit.unit, precision: unit.precision })} + + +
+ + +
+ ) : ( + + )} +
+ +
+ ) +}) diff --git a/venus-html5-app/src/app/KVNRV/components/Tanks/index.tsx b/venus-html5-app/src/app/KVNRV/components/Tanks/index.tsx new file mode 100644 index 0000000..39b64b7 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Tanks/index.tsx @@ -0,0 +1,5 @@ +export type TankProps = { + tankId: number +} + +export { SmallTank } from "./SmallTank" diff --git a/venus-html5-app/src/app/KVNRV/components/Views/Connecting/Connecting.js b/venus-html5-app/src/app/KVNRV/components/Views/Connecting/Connecting.js new file mode 100644 index 0000000..77aff5e --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Views/Connecting/Connecting.js @@ -0,0 +1,21 @@ +import React from "react" +import { Translate } from "react-i18nify" + +import "./Connecting.scss" + +const Connecting = () => ( +
+
+

+ +

+
+

.

+

.

+

.

+
+
+
+) + +export default Connecting diff --git a/venus-html5-app/src/app/KVNRV/components/Views/Connecting/Connecting.scss b/venus-html5-app/src/app/KVNRV/components/Views/Connecting/Connecting.scss new file mode 100644 index 0000000..1691c68 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Views/Connecting/Connecting.scss @@ -0,0 +1,40 @@ +@import "../../../css/variables"; + +.connecting { + display: flex; + flex-direction: column; + align-items: center; + margin-bottom: 3.5rem; +} + +.connecting__dots { + display: flex; + + .dot { + margin-left: 0.2rem; + margin-right: 0.2rem; + line-height: 0; + font-size: 3.5rem; + opacity: 0; + animation: showHideDot 1.2s ease-in-out infinite; + } + .two { + animation-delay: 0.25s; + } + .three { + animation-delay: 0.5s; + } +} + +@keyframes showHideDot { + 0% { + opacity: 0; + } + 40%, + 70% { + opacity: 1; + } + 100% { + opacity: 0; + } +} diff --git a/venus-html5-app/src/app/KVNRV/components/Views/Connecting/index.js b/venus-html5-app/src/app/KVNRV/components/Views/Connecting/index.js new file mode 100644 index 0000000..40434ab --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Views/Connecting/index.js @@ -0,0 +1 @@ +export { default } from "./Connecting" diff --git a/venus-html5-app/src/app/KVNRV/components/Views/Error/Error.scss b/venus-html5-app/src/app/KVNRV/components/Views/Error/Error.scss new file mode 100644 index 0000000..9349d10 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Views/Error/Error.scss @@ -0,0 +1,18 @@ +@import "../../../css/variables"; + +.error { + height: 100vh; + width: 100%; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + + div { + width: 50%; + } + + p { + margin-bottom: 10px; + } +} diff --git a/venus-html5-app/src/app/KVNRV/components/Views/Error/Error.tsx b/venus-html5-app/src/app/KVNRV/components/Views/Error/Error.tsx new file mode 100644 index 0000000..1e41e91 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Views/Error/Error.tsx @@ -0,0 +1,33 @@ +import React from "react" +import { Translate } from "react-i18nify" + +import "./Error.scss" + +const Error = ({ error }: { error?: any }) => { + return ( +
+
+

+ +

+

+ +

+

+ +

+

+ +

+

+ +

+
+
+ ) +} + +export default Error diff --git a/venus-html5-app/src/app/KVNRV/components/Views/Error/index.js b/venus-html5-app/src/app/KVNRV/components/Views/Error/index.js new file mode 100644 index 0000000..9f1cbb2 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Views/Error/index.js @@ -0,0 +1 @@ +export { default } from "./Error" diff --git a/venus-html5-app/src/app/KVNRV/components/Views/Metrics.tsx b/venus-html5-app/src/app/KVNRV/components/Views/Metrics.tsx new file mode 100644 index 0000000..2d3b33b --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Views/Metrics.tsx @@ -0,0 +1,233 @@ +import React, { useEffect, useRef, useState } from "react" +// @ts-ignore +import Hammer from "hammerjs" + +import Battery from "../Battery" +import DcLoads from "../DcLoads" +import PvCharger from "../PvCharger" +import AcLoads from "../AcLoads" +import Status from "../Status" +import AcMode from "../AcMode" +import Paginator from "../Paginator" +import { SmallTank } from "../Tanks" +import { SIZE_WIDE, SIZE_LONG } from "../../../components/Card" + +export const SCREEN_SIZES = { + MD: 499, + LG: 760, + XL: 1366, +} + +export const Metrics = () => { + let [pages, setPages] = useState(1) + let [currentPage, setCurrentPage] = useState(0) + let [layout, setLayout] = useState(<>) + const metricsRef = useRef(null) + // eslint-disable-next-line no-undef + let [hammer, setHammer] = useState(null!) + + useEffect(() => { + if (metricsRef.current) { + setHammer((prev) => { + if (prev) { + prev.destroy() + } + if (pages > 1) { + const newHammer = new Hammer.Manager(metricsRef.current!) + const Swipe = new Hammer.Swipe({ velocity: 0.2, direction: 2 | 4, threshold: 10 }) + newHammer.add(Swipe) + return newHammer + } else { + return prev + } + }) + } + }, [currentPage, pages]) + + useEffect(() => { + if (hammer) { + const next = () => { + setCurrentPage((currentPage) => (currentPage > 0 ? currentPage - 1 : currentPage)) + metricsRef.current!.style.marginLeft = "0" + } + const prev = () => { + setCurrentPage((currentPage) => (currentPage < pages - 1 ? currentPage + 1 : currentPage)) + metricsRef.current!.style.marginLeft = "0" + } + hammer.on("swiperight", next) + hammer.on("swipeleft", prev) + + // eslint-disable-next-line no-undef + hammer.on("hammer.input", (ev: HammerInput) => { + if (metricsRef.current) { + if (Math.abs(ev.deltaX) > Math.abs(ev.deltaY)) { + if ((ev.deltaX > 0 && currentPage !== 0) || (ev.deltaX < 0 && currentPage + 1 !== pages)) { + ;(metricsRef.current.children[0] as HTMLDivElement).style.transform = `translateX(${ev.deltaX}px)` + } + } + if (ev.isFinal) { + ;(metricsRef.current.children[0] as HTMLDivElement).style.transform = "translateX(0)" + } + } + }) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [hammer]) + + const computePages = () => { + // under 16/10 aspect ratio + const RATIO = 16 / 9 + const isNarrow = window.innerWidth / window.innerHeight > RATIO || window.innerHeight < 480 + const isTall = + !isNarrow && + window.innerWidth > SCREEN_SIZES.LG && + window.innerWidth < SCREEN_SIZES.XL && + window.innerHeight > 900 + + let pageNum = 3 + + if (isNarrow) { + if (window.innerWidth > SCREEN_SIZES.MD * RATIO) { + pageNum = 2 + } + if (window.innerWidth > SCREEN_SIZES.LG * RATIO) { + pageNum = 1 + } + } else { + if (window.innerWidth > SCREEN_SIZES.MD) { + pageNum = 2 + } + if (window.innerWidth > SCREEN_SIZES.LG || isTall) { + pageNum = 1 + } + } + + const currPage = Math.min(pageNum - 1, currentPage) + + if (isNarrow) { + setLayout( +
+ {isVisible(0, pageNum, currPage) && ( +
+ + +
+ )} + + {isVisible(1, pageNum, currPage) && ( +
+ +
+ + +
+
+ )} + + {isVisible(2, pageNum, currPage) && ( +
+
+ + +
+
+ + +
+
+ )} +
, + ) + } else { + setLayout( +
+ {isVisible(0, pageNum, currPage) && ( +
+ + +
+ + +
+ {isTall && ( +
+ + +
+ )} +
+ )} + + {isVisible(1, pageNum, currPage) && ( +
+ + + {isTall && ( +
+ + +
+ )} +
+ )} + + {!isTall && isVisible(2, pageNum, currPage) && ( +
+
+ + +
+
+ + +
+
+ )} +
, + ) + } + + setCurrentPage(currPage) + setPages(pageNum) + } + + useEffect(() => { + window.addEventListener("resize", computePages) + window.addEventListener("orientationchange", computePages) + setTimeout(() => computePages(), 200) + return () => { + window.removeEventListener("resize", computePages) + window.removeEventListener("orientationchange", computePages) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + useEffect(() => { + computePages() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [currentPage]) + + const isVisible = (elPage: number, pageNum: number, currPage: number) => { + if (pageNum === 2) { + if (currPage === 0 && elPage === 2) { + return false + } else if (currPage === 1 && elPage <= 1) { + return false + } + } else if (pageNum >= 3) { + if (elPage !== currPage) { + return false + } + } + return true + } + + return ( +
+ {layout} + {pages > 1 && } +
+ ) +} + +export default Metrics diff --git a/venus-html5-app/src/app/KVNRV/components/Views/MqttUnavailable.js b/venus-html5-app/src/app/KVNRV/components/Views/MqttUnavailable.js new file mode 100644 index 0000000..8af21ee --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Views/MqttUnavailable.js @@ -0,0 +1,30 @@ +import React from "react" +import { Translate } from "react-i18nify" +import Fade from "../../../components/Fade" +import MqttSettingsGuide from "../../images/mqtt-settings-v2.42.png" + +const MqttUnavailable = () => ( +
+ +
+
+
+
+
+ + + +
+
+ +
+
+
+ {"MQTT +
+
+ +
+) + +export default MqttUnavailable diff --git a/venus-html5-app/src/app/KVNRV/components/Views/RemoteConsole/RemoteConsole.scss b/venus-html5-app/src/app/KVNRV/components/Views/RemoteConsole/RemoteConsole.scss new file mode 100644 index 0000000..54ffae0 --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Views/RemoteConsole/RemoteConsole.scss @@ -0,0 +1,32 @@ +@import "../../../css/variables"; + +.remote-console__container { + margin-top: auto; + height: 85%; + display: flex; + align-items: center; + justify-content: center; + flex-grow: 1; + overflow: hidden; +} + +.remote-console { + height: 355px; + flex-basis: 800px; + overflow: hidden; + + @media #{$hide-remote-console-breakpoint} { + display: none; + } +} + +.remote-console__small_screen_info { + display: none; + + padding: 1rem; + text-align: center; + + @media #{$hide-remote-console-breakpoint} { + display: inline; + } +} diff --git a/venus-html5-app/src/app/KVNRV/components/Views/RemoteConsole/RemoteConsole.tsx b/venus-html5-app/src/app/KVNRV/components/Views/RemoteConsole/RemoteConsole.tsx new file mode 100644 index 0000000..2d6677b --- /dev/null +++ b/venus-html5-app/src/app/KVNRV/components/Views/RemoteConsole/RemoteConsole.tsx @@ -0,0 +1,25 @@ +import { translate, Translate } from "react-i18nify" + +import "./RemoteConsole.scss" + +type RemoteConsoleProps = { + onClickOutsideContainer: Function + host: string +} + +const RemoteConsole = ({ onClickOutsideContainer, host }: RemoteConsoleProps) => ( +
onClickOutsideContainer()}> +