diff --git a/.github/workflows/checkpr.yml b/.github/workflows/checkpr.yml index a5c06c8..250fdc2 100644 --- a/.github/workflows/checkpr.yml +++ b/.github/workflows/checkpr.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 @@ -25,4 +25,4 @@ jobs: - name: Scan security vulnerabilities with bandit run: bandit -c pyproject.toml -r . - name: Generate coverage report - run: pytest \ No newline at end of file + run: pytest diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7354a9f..f038c48 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 diff --git a/.vscode/settings.json b/.vscode/settings.json index ae37172..27824f8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -22,5 +22,6 @@ "python.analysis.addHoverSummaries": false, "python-envs.defaultEnvManager": "ms-python.python:venv", "python-envs.pythonProjects": [], - "python.defaultInterpreterPath": "${userHome}/pygpsclient/bin/python3" + "python.defaultInterpreterPath": "${userHome}/pygpsclient/bin/python3", + "python.testing.unittestEnabled": false } \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d477e7a..9757542 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ If you're intending to make significant changes, please raise them in the [Discu Being one of our contributors, you agree and confirm that: -* The work is all your own. +* The work is all your own. For the avoidance of doubt, this means **no AI coding agents such as Copilot**. * Your work will be distributed under a BSD 3-Clause License once your pull request is merged. * You submitted work fulfils or mostly fulfils our coding conventions, styles and standards. @@ -15,29 +15,21 @@ Please help us keep our issue list small by adding fixes: #{$ISSUE_NO} to the co ## Coding conventions * This is open source software. Code should be as simple and transparent as possible. Favour clarity over brevity. -* The code should be compatible with Python >= 3.9. +* The code should be compatible with Python >= 3.10. * Avoid external library dependencies unless there's a compelling reason not to. * We use and recommend [Visual Studio Code](https://code.visualstudio.com/) with the [Python Extension](https://marketplace.visualstudio.com/items?itemName=ms-python.python) for development and testing. * Code should be documented in accordance with [Sphinx](https://www.sphinx-doc.org/en/master/) docstring conventions. -* Code should formatted using [black](https://pypi.org/project/black/) (>= 24.4). -* We use and recommend [pylint](https://pypi.org/project/pylint/) (>=3.0.1) for code analysis. -* We use and recommend [bandit](https://pypi.org/project/bandit/) (>=1.7.5) for security vulnerability analysis. +* Code should formatted using [black](https://pypi.org/project/black/). +* We use and recommend [pylint](https://pypi.org/project/pylint/) for code analysis. +* We use and recommend [bandit](https://pypi.org/project/bandit/) for security vulnerability analysis. * Commits must be [signed](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits). ## Testing -While we endeavour to test on as wide a variety of u-blox devices as possible, as a volunteer project we only have a limited number of devices available. We particularly welcome testing contributions relating to specialised devices (e.g. high precision HP, real-time kinematics RTK, automotive dead-reckoning ADR, etc.). - We use python's native pytest framework for local unit testing, complemented by the GitHub Actions automated build and testing workflow. Please write pytest examples for new code you create and add them to the `/tests` folder following the naming convention `test_*.py`. -We test on the following platforms using a variety of u-blox devices from Generation 7 throught Generation 10: -* Windows 11 -* MacOS (Ventura & Sonoma, Intel & Apple Silicon) -* Linux (Ubuntu 22.04 LTS Jammy Jellyfish, 24.04 LTS Noble Numbat) -* Raspberry Pi OS (32-bit & 64-bit) - ## Submitting changes Please send a [GitHub Pull Request to pygnssutils](https://github.com/semuconsulting/pygnssutils/pulls) with a clear list of what you've done (read more about [pull requests](https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/about-pull-requests)). Please follow our coding conventions (above) and make sure all of your commits are atomic (one feature per commit). diff --git a/README.md b/README.md index 62229eb..49794e0 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ pygnssutils [gnssserver CLI](#gnssserver) | [gnssntripclient CLI](#gnssntripclient) | [gnssmqttclient CLI](#gnssmqttclient) | +[socketserver](#socketserver) | [RTK Demonstration](#rtkdemo) | [Troubleshooting](#troubleshooting) | [Graphical Client](#gui) | @@ -33,6 +34,7 @@ designated output stream. 1. `GNSSMQTTClient` class and its associated [`gnssmqttclient`](#gnssmqttclient) CLI utility. This implements a simple SPARTN IP (MQTT) Client which receives SPARTN correction data from an SPARTN IP location service and (optionally) sends this to a designated output stream. +1. `SocketServer` class based on the native Python `ThreadingTCPServer`. Capable of operating in two modes - Socket Server or NTRIP Caster. Provides two alternate client request handler classes - `ClientHandler` (HTTP) or `ClientHandlerTLS` (HTTPS). The pygnssutils homepage is located at [https://github.com/semuconsulting/pygnssutils](https://github.com/semuconsulting/pygnssutils). @@ -456,6 +458,31 @@ gnssmqttclient -h Refer to the [pyspartn documentation](https://github.com/semuconsulting/pyspartn?tab=readme-ov-file#reading) for further details on decrypting encrypted (`eaf=1`) SPARTN payloads. +--- +## SocketServer + +``` +class pygnssutils.socketserver.SocketServer(app, ntripmode, maxclients, msgqueue, **kwargs) +``` + +A helper class based on the native Python [`ThreadingTCPServer`](https://docs.python.org/3/library/socketserver.html) class, which streams GNSS data from an inbound message queue `msgqueue` to a maximum of `maxclients` TCP clients. + + Capable of operating in either of two modes, according to the `ntripmode` argument: + 1. ntripmode = 0 - Socket Server; streams incoming GNSS data to any TCP client, without authentication. + 2. ntripmode = 1 - NTRIP Caster; acts as a simple NTRIP Server/Caster, streaming incoming RTCM3 data to any authenticated NTRIP client. + + Provides two client request handler classes: + - `ClientHandler` - unencrypted HTTP connection. + - `ClientHandlerTLS` - encrypted HTTPS (TLS) connection. + + **NB:** HTTPS requires a valid x509 TLS certificate/key pair (in pem format) to be located at a path designated by environment variable `PYGNSSUTILS_PEMPATH`. The default path is `$HOME\pygnssutils.pem`. The following openssl command can be used to create a suitable pem file for test and demonstration purposes: + + ```shell + openssl req -x509 -newkey rsa:4096 -keyout pygnssutils.pem -out pygnssutils.pem -sha256 -days 3650 -nodes + ``` + +Refer to the [Sphinx API documentation](https://www.semuconsulting.com/pygnssutils/pygnssutils.html#module-pygnssutils.socket_server) for further details. + --- ## NTRIP RTK demonstration using `gnssserver` and `gnssntripclient` @@ -531,7 +558,6 @@ gnssntripclient --server 192.168.0.27 --port 2101 --https 0 --mountpoint pygnssu Refer to [cryptography installation README.md](https://github.com/semuconsulting/pyspartn/blob/main/cryptography_installation/README.md). - --- ## Graphical Client diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 2ac27c7..8e686d3 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,9 +1,22 @@ # pygnssutils +### RELEASE 1.1.19 + +ENHANCEMENTS: + +1. Add support for TLS connections in SocketServer. Introduces two alternative client request handler classes - ClientHandler (HTTP) or ClientHandlerTLS (HTTPS). TLS operation requires a suitable TLS certificate/key pair (in pem format) to be located at a path designated by +environment variable `PYGNSSUTILS_PEMPATH` - the default path is $HOME/pygnssutils.pem. See Sphinx documentation for details. + + A self-signed pem file suitable for test and demonstration purposes can be created thus: + ```shell + openssl req -x509 -newkey rsa:4096 -keyout pygnssutils.pem -out pygnssutils.pem -sha256 -days 3650 -nodes + ``` + ### RELEASE 1.1.18 ENHANCEMENTS: +1. Add gnssreader class. 1. Add support for Quectel QGC protocol. ### RELEASE 1.1.17 diff --git a/pyproject.toml b/pyproject.toml index a693937..7558ff4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,11 +37,11 @@ dependencies = [ "certifi>=2025.0.0", "paho-mqtt>=2.1.0", "pyserial>=3.5", - "pyspartn>=1.0.7", - "pyubx2>=1.2.58", - "pysbf2>=1.0.0", - "pyubxutils>=1.0.3", - "pyqgc>=0.1.1", + "pyspartn>=1.0.8", + "pyubx2>=1.2.59", + "pysbf2>=1.0.3", + "pyubxutils>=1.0.5", + "pyqgc>=0.1.2", ] [project.scripts] diff --git a/src/pygnssutils/__init__.py b/src/pygnssutils/__init__.py index 27667a5..64ab34d 100644 --- a/src/pygnssutils/__init__.py +++ b/src/pygnssutils/__init__.py @@ -14,6 +14,14 @@ from pygnssutils.globals import * from pygnssutils.gnssmqttclient import GNSSMQTTClient from pygnssutils.gnssntripclient import GNSSNTRIPClient +from pygnssutils.gnssreader import ( + NMEA_PROTOCOL, + QGC_PROTOCOL, + RTCM3_PROTOCOL, + SBF_PROTOCOL, + UBX_PROTOCOL, + GNSSReader, +) from pygnssutils.gnssserver import GNSSSocketServer from pygnssutils.gnssstreamer import GNSSStreamer from pygnssutils.helpers import * diff --git a/src/pygnssutils/_version.py b/src/pygnssutils/_version.py index 0682ba0..8919961 100644 --- a/src/pygnssutils/_version.py +++ b/src/pygnssutils/_version.py @@ -8,4 +8,4 @@ :license: BSD 3-Clause """ -__version__ = "1.1.18" +__version__ = "1.1.19" diff --git a/src/pygnssutils/globals.py b/src/pygnssutils/globals.py index fe12672..2905c1e 100644 --- a/src/pygnssutils/globals.py +++ b/src/pygnssutils/globals.py @@ -8,6 +8,10 @@ :license: BSD 3-Clause """ +OKCOL = "green" +ERRCOL = "salmon" +INFOCOL = "steelblue2" + CLIAPP = "CLI" DEFAULT_BUFSIZE = 4096 # buffer size for NTRIP client """Default socket buffer size""" @@ -74,6 +78,8 @@ """Serial output""" OUTPUT_SOCKET = 3 """Socket output""" +OUTPUT_SOCKET_TLS = 6 +"""Socket output with TLS""" OUTPUT_HANDLER = 4 """Custom output handler""" OUTPUT_TEXT_FILE = 5 @@ -102,7 +108,7 @@ """Disconnected""" CONNECTED = 1 """Connected""" -MAXCONNECTION = 2 +MAXCONNECTION = 5 """Maximum connections reached (for socket server)""" LOGFORMAT = "{asctime}.{msecs:.0f} - {levelname} - {name} - {message}" """Logging format""" @@ -111,7 +117,7 @@ NOGGA = -1 """No GGA sentence to be sent (for NTRIP caster)""" EPILOG = ( - "© 2022 SEMU Consulting BSD 3-Clause license" + "© 2022 semuadmin (Steve Smith) BSD 3-Clause license" " - https://github.com/semuconsulting/pygnssutils/" ) """CLI argument parser epilog""" @@ -195,7 +201,10 @@ MINMMEA_ID = [b"\xf0\x00", b"\xf0\x02", b"\xf0\x03", b"\xf0\x04", b"\xf0\x05"] ALLUBX_CLS = [b"\x01"] MINUBX_ID = [b"\x01\x04", b"\x01\x07", b"\x01\x35"] - +PYGNSSUTILS_PEM = "pygnssutils.pem" +"""Name of default TLS PEM file""" +PYGNSSUTILS_PEMPATH = "PYGNSSUTILS_PEMPATH" +"""Name of environment variable containing path to TLS PEM file""" TOPIC_KEY = "/pp/ubx/0236/{}" TOPIC_ASSIST = "/pp/ubx/mga" TOPIC_DATA = "/pp/{}/{}" diff --git a/src/pygnssutils/gnssmqttclient_cli.py b/src/pygnssutils/gnssmqttclient_cli.py index db6b45a..1a126b4 100644 --- a/src/pygnssutils/gnssmqttclient_cli.py +++ b/src/pygnssutils/gnssmqttclient_cli.py @@ -25,11 +25,14 @@ ENV_MQTT_CLIENTID, ENV_MQTT_KEY, EPILOG, + MAXCONNECTION, + NTRIP2, OUTPORT_SPARTN, OUTPUT_FILE, OUTPUT_NONE, OUTPUT_SERIAL, OUTPUT_SOCKET, + OUTPUT_SOCKET_TLS, SPARTN_PPSERVER, ) from pygnssutils.gnssmqttclient import TIMEOUT, GNSSMQTTClient @@ -195,10 +198,17 @@ def main(): f"CLI output type {OUTPUT_NONE} = none, " f"{OUTPUT_FILE} = binary file, " f"{OUTPUT_SERIAL} = serial port, " - f"{OUTPUT_SOCKET} = TCP socket server" + f"{OUTPUT_SOCKET} = TCP socket server, " + f"{OUTPUT_SOCKET_TLS} = TCP socket server with TLS" ), type=int, - choices=[OUTPUT_NONE, OUTPUT_FILE, OUTPUT_SERIAL, OUTPUT_SOCKET], + choices=[ + OUTPUT_NONE, + OUTPUT_FILE, + OUTPUT_SERIAL, + OUTPUT_SOCKET, + OUTPUT_SOCKET_TLS, + ], default=OUTPUT_NONE, ) ap.add_argument( @@ -208,7 +218,8 @@ def main(): "Output medium as formatted string. " f"If clioutput = {OUTPUT_FILE}, format = file name (e.g. '/home/myuser/spartn.log'); " f"If clioutput = {OUTPUT_SERIAL}, format = port@baudrate (e.g. '/dev/tty.ACM0@38400'); " - f"If clioutput = {OUTPUT_SOCKET}, format = hostip:port (e.g. '0.0.0.0:50010'). " + f"If clioutput = {OUTPUT_SOCKET} or {OUTPUT_SOCKET_TLS}, " + "format = hostip:port (e.g. '0.0.0.0:50010'). " "NB: gnssmqttclient will have exclusive use of any serial or server port." ), default=None, @@ -231,14 +242,15 @@ def main(): with Serial(port, int(baud), timeout=3) as output: kwargs["output"] = output runclient(**kwargs) - elif cliout == OUTPUT_SOCKET: + elif cliout in (OUTPUT_SOCKET, OUTPUT_SOCKET_TLS): host, port = kwargs["output"].split(":") + tls = cliout == OUTPUT_SOCKET_TLS kwargs["output"] = Queue() # socket server runs as background thread, piping # output from mqtt client via a message queue Thread( target=runserver, - args=(host, int(port), kwargs["output"]), + args=(host, int(port), kwargs["output"], 0, MAXCONNECTION, tls, NTRIP2), daemon=True, ).start() runclient(**kwargs) diff --git a/src/pygnssutils/gnssntripclient.py b/src/pygnssutils/gnssntripclient.py index 82b2cd7..e5ffbaf 100644 --- a/src/pygnssutils/gnssntripclient.py +++ b/src/pygnssutils/gnssntripclient.py @@ -56,8 +56,10 @@ ENV_MQTT_KEY, ENV_NTRIP_PASSWORD, ENV_NTRIP_USER, + ERRCOL, FIXES, HTTPCODES, + INFOCOL, MAXPORT, NOGGA, NTRIP2, @@ -65,7 +67,7 @@ OUTPORT_NTRIP, VERBOSITY_MEDIUM, ) -from pygnssutils.helpers import find_mp_distance, ipprot2int, set_logging +from pygnssutils.helpers import check_pemfile, find_mp_distance, ipprot2int, set_logging TIMEOUT = 3 GGALIVE = 0 @@ -116,7 +118,7 @@ def __init__( self._timeout = int(kwargs.pop("timeout", INACTIVITY_TIMEOUT)) except (ParameterError, ValueError, TypeError) as err: msg = f"Invalid input arguments {err}" - self._app_update_status(False, (str(err), "red")) + self._app_update_status(False, (str(err), ERRCOL)) raise ParameterError(msg + "\nType gnssntripclient -h for help.") from err self._connected = False @@ -159,7 +161,8 @@ def run(self, **kwargs) -> bool: :param str server: (kwarg) NTRIP server URL ("") :param int port: (kwarg) NTRIP port (2101) - :param int https: (kwarg) HTTPS (TLS) connection? 0 = HTTP 1 = HTTPS (0) + :param int https: (kwarg) Enable HTTPS (TLS) connection? (0) + :param int selfsign: (kwarg) Allow self-sign TLS certificate (0) :param str mountpoint: (kwarg) NTRIP mountpoint ("", leave blank to get sourcetable) :param str datatype: (kwarg) Data type - RTCM or SPARTN ("RTCM") :param str version: (kwarg) NTRIP protocol version ("2.0") @@ -195,7 +198,7 @@ def run(self, **kwargs) -> bool: except (ParameterError, ValueError, TypeError) as err: msg = f"Invalid input arguments - {err}" - self._app_update_status(False, (str(err), "red")) + self._app_update_status(False, (str(err), ERRCOL)) raise ParameterError(msg + "\nType gnssntripclient -h for help.") from err self._connected = True @@ -274,7 +277,7 @@ def _read_thread( f". Retrying in {self._retryinterval * (2**self._retrycount)} secs " f"({self._retrycount}/{self._retries}) ..." ) - self._app_update_status(True, (errm, "red")) + self._app_update_status(True, (errm, ERRCOL)) # critical errors... except (ssl.SSLError, ssl.SSLCertVerificationError) as err: errc = err.strerror @@ -292,7 +295,7 @@ def _read_thread( if errc != "": # break connection on critical error self.stop() - self._app_update_status(False, (errc, "red")) + self._app_update_status(False, (errc, ERRCOL)) break if not self._stopevent.is_set() and not self._sleepevent.is_set(): @@ -319,8 +322,12 @@ def _open_connection(self, settings: dict) -> socket: if int(settings["https"]): context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) context.load_verify_locations(findcacerts()) + if int(settings.get("selfsign", 0)): + pem, exists = check_pemfile() + # context.verify_mode = ssl.CERT_NONE + context.load_verify_locations(pem) + context.check_hostname = False sock = context.wrap_socket(sock, server_hostname=hostname) - return sock def _close_connection(self, sock: socket): @@ -375,7 +382,7 @@ def _do_request( f"Streaming {settings['datatype']} data from " f"{settings['server']}:{settings['port']}/{settings['mountpoint']} ..." ) - self._app_update_status(True, (msg, "blue")) + self._app_update_status(True, (msg, INFOCOL)) self._parse_ntrip_data( sock, settings, @@ -390,14 +397,14 @@ def _do_request( f"Connection failed {self._response_status['code']} " f"{self._response_status['description']}" ) - self._app_update_status(False, (msg, "red")) + self._app_update_status(False, (msg, ERRCOL)) return 0 if self.is_sourcetable: stable = self._parse_sourcetable(self.response_body) self._settings["sourcetable"] = stable mp, dist = self._get_closest_mountpoint() self._do_output(output, stable, (mp, dist)) - self._app_update_status(False, ("Sourcetable retrieved", "blue")) + self._app_update_status(False, ("Sourcetable retrieved", INFOCOL)) return 0 return 1 @@ -711,7 +718,7 @@ def _app_update_status(self, status: bool, msgt: tuple = None): :param tuple msgt: (message, color) """ - if msgt[1] == "red": + if msgt[1] == ERRCOL: self.logger.error(msgt[0]) else: self.logger.info(msgt[0]) @@ -783,6 +790,7 @@ def settings(self, kwargs: dict): self._settings["server"] = kwargs.get("server", "") self._settings["port"] = int(kwargs.get("port", OUTPORT_NTRIP)) self._settings["https"] = int(kwargs.get("https", 0)) + self._settings["selfsign"] = int(kwargs.get("selfsign", 0)) self._settings["flowinfo"] = int(kwargs.get("flowinfo", 0)) self._settings["scopeid"] = int(kwargs.get("scopeid", 0)) self._settings["mountpoint"] = kwargs.get("mountpoint", "") diff --git a/src/pygnssutils/gnssntripclient_cli.py b/src/pygnssutils/gnssntripclient_cli.py index 7bd085a..b2994e1 100644 --- a/src/pygnssutils/gnssntripclient_cli.py +++ b/src/pygnssutils/gnssntripclient_cli.py @@ -26,12 +26,14 @@ ENV_NTRIP_PASSWORD, ENV_NTRIP_USER, EPILOG, + MAXCONNECTION, NTRIP1, NTRIP2, OUTPUT_FILE, OUTPUT_NONE, OUTPUT_SERIAL, OUTPUT_SOCKET, + OUTPUT_SOCKET_TLS, ) from pygnssutils.gnssntripclient import ( GGAFIXED, @@ -96,6 +98,14 @@ def main(): choices=[0, 1], default=0, ) + ap.add_argument( + "--selfsign", + required=False, + help=("Allow self-signed TLS certificate?"), + type=int, + choices=[0, 1], + default=0, + ), ap.add_argument( "-I", "--ipprot", @@ -204,10 +214,17 @@ def main(): f"CLI output type {OUTPUT_NONE} = none, " f"{OUTPUT_FILE} = binary file, " f"{OUTPUT_SERIAL} = serial port, " - f"{OUTPUT_SOCKET} = TCP socket server" + f"{OUTPUT_SOCKET} = TCP socket server, " + f"{OUTPUT_SOCKET_TLS} = TCP socket server with TLS" ), type=int, - choices=[OUTPUT_NONE, OUTPUT_FILE, OUTPUT_SERIAL, OUTPUT_SOCKET], + choices=[ + OUTPUT_NONE, + OUTPUT_FILE, + OUTPUT_SERIAL, + OUTPUT_SOCKET, + OUTPUT_SOCKET_TLS, + ], default=OUTPUT_NONE, ) ap.add_argument( @@ -217,7 +234,8 @@ def main(): f"Output medium as formatted string. " f"If clioutput = {OUTPUT_FILE}, format = file name (e.g. '/home/myuser/rtcm.log'); " f"If clioutput = {OUTPUT_SERIAL}, format = port@baudrate (e.g. '/dev/tty.ACM0@38400'); " - f"If clioutput = {OUTPUT_SOCKET}, format = hostip:port (e.g. '0.0.0.0:50010'). " + f"If clioutput = {OUTPUT_SOCKET} or {OUTPUT_SOCKET_TLS}, " + "format = hostip:port (e.g. '0.0.0.0:50010'). " "NB: gnssntripclient will have exclusive use of any serial or server port." ), default=None, @@ -237,14 +255,15 @@ def main(): with Serial(port, int(baud), timeout=3) as output: kwargs["output"] = output runclient(**kwargs) - elif cliout == OUTPUT_SOCKET: + elif cliout in (OUTPUT_SOCKET, OUTPUT_SOCKET_TLS): host, port = kwargs["output"].split(":") + tls = cliout == OUTPUT_SOCKET_TLS kwargs["output"] = Queue() # socket server runs as background thread, piping # output from ntrip client via a message queue Thread( target=runserver, - args=(host, int(port), kwargs["output"]), + args=(host, int(port), kwargs["output"], 0, MAXCONNECTION, tls, NTRIP2), daemon=True, ).start() runclient(**kwargs) diff --git a/src/pygnssutils/gnssserver.py b/src/pygnssutils/gnssserver.py index 1988adf..0cf960a 100644 --- a/src/pygnssutils/gnssserver.py +++ b/src/pygnssutils/gnssserver.py @@ -24,7 +24,7 @@ from pygnssutils.globals import NTRIP2, OUTPORT from pygnssutils.gnssstreamer import GNSSStreamer from pygnssutils.helpers import format_conn, ipprot2int -from pygnssutils.socket_server import ClientHandler, SocketServer +from pygnssutils.socket_server import ClientHandler, ClientHandlerTLS, SocketServer class GNSSSocketServer: @@ -39,6 +39,7 @@ def __init__( ipprot: str = "IPv4", hostip: str = "0.0.0.0", outport: int = OUTPORT, + tls: bool = 0, maxclients: int = 5, ntripmode: int = 0, ntripversion: str = NTRIP2, @@ -58,6 +59,7 @@ def __init__( :param str ipprot: IP protocol IPv4/IPv6 ("IPv4") :param int hostip: host ip address (0.0.0.0) :param int outport: TCP port (50010) + :param bool tls: Enable TLS (HTTPS) (0) :param int maxclients: maximum number of connected clients (5) :param int ntripmode: 0 = socket server, 1 - NTRIP server (0) :param str ntripversion: NTRIP version "1.0"/"2.0" ("2.0") @@ -80,6 +82,7 @@ def __init__( self._ntrippassword = ntrippassword self._hostip = hostip self._outport = int(outport) + self._tls = tls self._maxclients = int(maxclients) self._kwargs = kwargs self._output = Queue() @@ -176,6 +179,7 @@ def _start_output_thread(self) -> Thread: self._ipprot, self._hostip, self._outport, + self._tls, self._ntripmode, self._maxclients, self._output, @@ -208,6 +212,7 @@ def _output_thread( ipprot: str, hostip: str, outport: int, + tls: bool, ntripmode: int, maxclients: int, output: object, @@ -223,6 +228,7 @@ def _output_thread( :param str ipprot: IP protocol :param int hostip: host ip address :param str outport: TCP port + :param bool tls: Use TLS :param int maxclients: maximum number of connected clients :param int ntripmode: :param str ntripversion: NTRIP version @@ -230,6 +236,7 @@ def _output_thread( :param str ntrippassword: NTRIP caster authentication password """ + requesthandler = ClientHandlerTLS if tls else ClientHandler try: conn = format_conn(ipprot2int(ipprot), hostip, outport) with SocketServer( @@ -238,7 +245,7 @@ def _output_thread( maxclients, output, conn, - ClientHandler, + requesthandler, ntripversion=ntripversion, ntripuser=ntripuser, ntrippassword=ntrippassword, diff --git a/src/pygnssutils/gnssserver_cli.py b/src/pygnssutils/gnssserver_cli.py index c6d3ebe..fb977de 100644 --- a/src/pygnssutils/gnssserver_cli.py +++ b/src/pygnssutils/gnssserver_cli.py @@ -111,6 +111,14 @@ def main(): type=int, default=50010, ) + ap.add_argument( + "-T", + "--tls", + required=False, + help="Enable TLS (HTTPS) - set PYGNSSUTILS_PEMPATH", + type=bool, + default=0, + ) ap.add_argument( "--ipprot", required=False, diff --git a/src/pygnssutils/gnssstreamer.py b/src/pygnssutils/gnssstreamer.py index b741326..00db3b4 100644 --- a/src/pygnssutils/gnssstreamer.py +++ b/src/pygnssutils/gnssstreamer.py @@ -340,7 +340,7 @@ def _read_loop( raise ParameterError() from err except OSError: # thread terminated while reading break - except Exception as err: # pylint disable=broad-exception-caught + except Exception as err: # pylint: disable=broad-exception-caught self._errcount += 1 self.logger.error(f"Error parsing data stream {err}") continue diff --git a/src/pygnssutils/gnssstreamer_cli.py b/src/pygnssutils/gnssstreamer_cli.py index 47d4dc1..a4897f9 100644 --- a/src/pygnssutils/gnssstreamer_cli.py +++ b/src/pygnssutils/gnssstreamer_cli.py @@ -62,11 +62,14 @@ INPUT_NTRIP_RTCM, INPUT_NTRIP_SPARTN, INPUT_SERIAL, + MAXCONNECTION, + NTRIP2, OUTPUT_FILE, OUTPUT_HANDLER, OUTPUT_NONE, OUTPUT_SERIAL, OUTPUT_SOCKET, + OUTPUT_SOCKET_TLS, OUTPUT_TEXT_FILE, UBXSIMULATOR, ) @@ -267,14 +270,15 @@ def _setup_output(**kwargs): with Serial(port, int(baud), timeout=3) as output: kwargs["output"] = output _setup_datastream(**kwargs) - elif cliout == OUTPUT_SOCKET: + elif cliout in (OUTPUT_SOCKET, OUTPUT_SOCKET_TLS): host, port = output.split(":") + tls = cliout == OUTPUT_SOCKET_TLS output = Queue() # socket server runs as background thread, piping # output from gnssstreamer via a message queue Thread( target=runserver, - args=(host, int(port), output), + args=(host, int(port), output, 0, MAXCONNECTION, tls, NTRIP2), daemon=True, ).start() kwargs["output"] = output @@ -504,6 +508,7 @@ def main(): f"{OUTPUT_FILE} = binary file, " f"{OUTPUT_SERIAL} = serial port, " f"{OUTPUT_SOCKET} = TCP socket server, " + f"{OUTPUT_SOCKET_TLS} = TCP socket server with TLS, " f"{OUTPUT_HANDLER} = evaluable Python expression, " f"{OUTPUT_TEXT_FILE} = text file" ), @@ -513,6 +518,7 @@ def main(): OUTPUT_FILE, OUTPUT_SERIAL, OUTPUT_SOCKET, + OUTPUT_SOCKET_TLS, OUTPUT_HANDLER, OUTPUT_TEXT_FILE, ], @@ -526,7 +532,8 @@ def main(): f"If clioutput = {OUTPUT_FILE} or {OUTPUT_TEXT_FILE}, format = file name " "(e.g. '/home/myuser/ubxdata.ubx'); " f"If clioutput = {OUTPUT_SERIAL}, format = port@baudrate (e.g. '/dev/tty.ACM0@38400'); " - f"If clioutput = {OUTPUT_SOCKET}, format = hostip:port (e.g. '0.0.0.0:50010'); " + f"If clioutput = {OUTPUT_SOCKET} or {OUTPUT_SOCKET_TLS}, " + "format = hostip:port (e.g. '0.0.0.0:50010'); " f"If clioutput = {OUTPUT_HANDLER}, format = evaluable Python expression. " "NB: gnssstreamer will have exclusive use of any serial or server port." ), diff --git a/src/pygnssutils/helpers.py b/src/pygnssutils/helpers.py index 25065d3..85c6dd3 100644 --- a/src/pygnssutils/helpers.py +++ b/src/pygnssutils/helpers.py @@ -13,8 +13,10 @@ import logging import logging.handlers from argparse import ArgumentParser +from datetime import datetime, timezone from math import cos, radians, sin -from os import getenv +from os import getenv, path +from pathlib import Path from socket import AF_INET, AF_INET6, gaierror, getaddrinfo from pynmeagps import haversine @@ -25,6 +27,8 @@ LOGFORMAT, LOGGING_LEVELS, LOGLIMIT, + PYGNSSUTILS_PEM, + PYGNSSUTILS_PEMPATH, VERBOSITY_CRITICAL, VERBOSITY_DEBUG, VERBOSITY_HIGH, @@ -269,6 +273,20 @@ def cel2cart(elevation: float, azimuth: float) -> tuple: return (x, y) +def format_dates() -> tuple: + """ + Format response header dates. + + :returns: tuple of (http_date, server_date) + :rtype: tuple + """ + + dat = datetime.now(timezone.utc) + http_date = dat.strftime("%a, %d %b %Y %H:%M:%S %Z") + server_date = dat.strftime("%d %b %Y") + return (http_date, server_date) + + def format_json(message: object) -> str: """ Format object as JSON document. @@ -318,16 +336,15 @@ def format_conn( """ if family == AF_INET6: - if family == AF_INET6: - if flowinfo != 0 or scopeid != 0: - return (server, port, flowinfo, scopeid) - try: - gai = getaddrinfo(server, port) - if len(gai) == 1: # No IP6 support (Windows) - return gai[0][4] - return gai[1][4] # IP6 support (Posix) - except gaierror as err: - raise ValueError(f"Invalid server or port {server} {port}") from err + if flowinfo != 0 or scopeid != 0: + return (server, port, flowinfo, scopeid) + try: + gai = getaddrinfo(server, port) + if len(gai) == 1: # No IP6 support (Windows) + return gai[0][4] + return gai[1][4] # IP6 support (Posix) + except gaierror as err: + raise ValueError(f"Invalid server or port {server} {port}") from err if family == AF_INET: return (server, port) raise ValueError(f"Invalid family value {family}") @@ -398,9 +415,9 @@ def parse_url(url: str) -> tuple: hostpath = host.split("/", 1) if len(hostpath) == 1: hostname = hostpath[0] - path = "/" + pth = "/" else: - hostname, path = hostpath + hostname, pth = hostpath hostport = hostname.split(":", 1) if len(hostport) == 1: @@ -413,4 +430,16 @@ def parse_url(url: str) -> tuple: except (ValueError, IndexError) as err: raise ParameterError(f"Invalid URL {url} {err}") from err - return prot, hostname, port, path + return prot, hostname, port, pth + + +def check_pemfile() -> tuple: + """ + Check TLS PEM (Certificate/Key) file for HTTPS server connection. + + :return: tuple of (path to PEM file, exists flag) + :rtype: tuple + """ + + pem = getenv(PYGNSSUTILS_PEMPATH, default=path.join(Path.home(), PYGNSSUTILS_PEM)) + return pem, Path(pem).exists() diff --git a/src/pygnssutils/socket_server.py b/src/pygnssutils/socket_server.py index e1440a2..0be8140 100644 --- a/src/pygnssutils/socket_server.py +++ b/src/pygnssutils/socket_server.py @@ -1,30 +1,41 @@ """ -TCP socket server for PyGPSClient application. - -(could also be used independently of a tkinter app framework) +TCP socket server for GNSS applications. Reads raw data from GNSS receiver message queue and outputs this to multiple TCP socket clients. -Operates in two modes according to ntripmode setting: - -0 - open socket mode - will stream GNSS data to any connected client - without authentication. -1 - NTRIP caster mode - implements NTRIP server protocol and will - respond to NTRIP client authentication, sourcetable and RTCM3 data - stream requests. - NB: THIS ASSUMES THE CONNECTED GNSS RECEIVER IS OPERATING IN BASE - STATION (SURVEY-IN OR FIXED) MODE AND OUTPUTTING THE RELEVANT RTCM3 MESSAGES. - -For NTRIP caster mode, authorization credentials can be supplied via keyword -arguments or set as environment variables: -export PYGPSCLIENT_USER="user" -export PYGPSCLIENT_PASSWORD="password" - NB: This utility is used by PyGPSClient - do not change footprint of any public methods without first checking impact on PyGPSClient - https://github.com/semuconsulting/PyGPSClient. +Provides two client request handler classes: + +- ClientHandler - HTTP connection +- ClientHandlerTLS - HTTPS (TLS) connection + + TLS requires a valid TLS certificate/key pair (in pem format) + to be located at a path set in environment variable PYGNSSUTILS_PEMPATH. + The default path is $HOME/pygnssutils.pem. + + A pem file suitable for demo and test purposes can be created thus:: + + openssl req -x509 -newkey rsa:4096 -keyout host.pem -out host.pem -sha256 -days 3650 -nodes + +For TLS (HTTPS) operation, instantiate SocketServer using a request handler +of ClientHandlerTLS rather than ClientHander. + +Operates in either of two modes according to ntripmode setting: + +- ntripmode=0. Open socket server mode - will stream GNSS data to any connected client + without authentication. +- ntripmode=1. NTRIP caster mode - implements NTRIP server protocol and will + respond to NTRIP client authentication, sourcetable and RTCM3 data stream requests. + NB: THIS ASSUMES THE CONNECTED GNSS RECEIVER IS OPERATING IN BASE + STATION MODE (SURVEY-IN OR FIXED) AND OUTPUTTING THE RELEVANT RTCM3 MESSAGES. + +For NTRIP caster mode, authorization credentials can be supplied via keyword +arguments or set via environment variables PYGPSCLIENT_USER and PYGPSCLIENT_PASSWORD + Created on 16 May 2022 :author: semuadmin (Steve Smith) @@ -33,11 +44,11 @@ """ from base64 import b64encode -from datetime import datetime, timezone from logging import getLogger from os import getenv from queue import Queue from socketserver import StreamRequestHandler, ThreadingTCPServer +from ssl import CERT_OPTIONAL, PROTOCOL_TLS, SSLContext from threading import Event, Thread from pygnssutils._version import __version__ as VERSION @@ -48,11 +59,12 @@ ENV_NTRIP_PASSWORD, ENV_NTRIP_USER, MAXCONNECTION, + NTRIP1, NTRIP2, PYGPSMP, RTCMTYPES, ) -from pygnssutils.helpers import ipprot2int +from pygnssutils.helpers import check_pemfile, format_dates, ipprot2int # from pygpsclient import version as PYGPSVERSION @@ -270,7 +282,7 @@ class ClientHandler(StreamRequestHandler): Threaded TCP client connection handler class. """ - def __init__(self, *args, **kwargs): + def __init__(self, request, client_address, server): """ Overridden constructor. """ @@ -279,7 +291,7 @@ def __init__(self, *args, **kwargs): self._msgqueue = None self._allowed = False - super().__init__(*args, **kwargs) + super().__init__(request, client_address, server) def setup(self, *args, **kwargs): """ @@ -380,9 +392,9 @@ def _process_ntrip_request(self, data: bytes) -> bytes: elif mountpoint == f"/{PYGPSMP}": # valid mountpoint validmp = True - http_date, server_date = self._format_dates() + http_date, server_date = format_dates() if not authorized: # respond with 401 - if self.server.ntripversion == "1.0": + if self.server.ntripversion == NTRIP1: http = ( "HTTP/1.1 401 Unauthorized\r\n" f"Date: {http_date}\r\n" @@ -423,7 +435,7 @@ def _format_sourcetable(self) -> str: :rtype: str """ - http_date, server_date = self._format_dates() + http_date, server_date = format_dates() lat, lon = self.server.latlon ipaddr, port = self.server.server_address pygu = PYGPSMP.upper() @@ -439,7 +451,7 @@ def _format_sourcetable(self) -> str: f"2;GPS+GLO+GAL+BDS;{pygu};GBR;{lat};{lon};0;0;{pygu};none;B;N;0;\r\n" "ENDSOURCETABLE\r\n" ) - if self.server.ntripversion == "1.0": + if self.server.ntripversion == NTRIP1: http = ( "SOURCETABLE 200 OK\r\n" f"Date: {http_date}\r\n" @@ -472,8 +484,8 @@ def _format_data(self) -> str: :rtype: str """ - http_date, server_date = self._format_dates() - if self.server.ntripversion == "1.0": + http_date, server_date = format_dates() + if self.server.ntripversion == NTRIP1: http = "ICY 200 OK\r\n\r\n" else: http = ( @@ -489,19 +501,6 @@ def _format_data(self) -> str: ) return http - def _format_dates(self) -> tuple: - """ - Format response header dates. - - :returns: tuple of (http_date, server_date) - :rtype: tuple - """ - - dat = datetime.now(timezone.utc) - http_date = dat.strftime("%a, %d %b %Y %H:%M:%S %Z") - server_date = dat.strftime("%d %b %Y") - return (http_date, server_date) - def _write_from_mq(self): """ Get data from message queue and write to socket. @@ -513,12 +512,37 @@ def _write_from_mq(self): self.wfile.flush() +class ClientHandlerTLS(ClientHandler): + """ + Threaded TCP client connection handler class with TLS (HTTPS). + """ + + def __init__(self, request, client_address, server): + """ + Overridden constructor. + """ + + self._qidx = None + self._msgqueue = None + self._allowed = False + + pem, exists = check_pemfile() + context = SSLContext(PROTOCOL_TLS) + context.load_cert_chain(certfile=pem) + context.verify_mode = CERT_OPTIONAL + context.check_hostname = False + request = context.wrap_socket(request, server_side=True) + super().__init__(request, client_address, server) + + def runserver( host: str, port: int, mq: Queue, ntripmode: int = 0, - maxclients: int = 5, + maxclients: int = MAXCONNECTION, + tls: bool = False, + ntripversion: str = NTRIP2, **kwargs, ): """ @@ -529,18 +553,20 @@ def runserver( :param int port: port :param Queue mq: output message queue :param int ntripmode: 0 = basic, 1 = ntrip caster - :param int maxclients: max concurrent clients - :param str ntripversion: (kwarg) NTRIP version 1.0 or 2.0 + :param int maxclients: max concurrent clients (5) + :param bool tls: Enable TLS (HTTPS) (0) (remember to set PYGNSSUTILS_PEMPATH) + :param str ntripversion: NTRIP version 1.0 or 2.0 """ + requesthandler = ClientHandlerTLS if tls else ClientHandler with SocketServer( CLIAPP, ntripmode, maxclients, mq, # message queue containing raw data from source (host, port), - ClientHandler, - ntripversion=kwargs.get("ntripversion", NTRIP2), + requesthandler, + ntripversion=ntripversion, ntripuser=kwargs.get("ntripuser", "anon"), ntrippassword=kwargs.get("ntrippassword", "password"), ) as server: diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..c375539 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,66 @@ +""" +Helper, Property and Static method tests for pygnssutils + +Created on 26 May 2022 + +*** NB: must be saved in UTF-8 format *** + +@author: semuadmin +""" + +# pylint: disable=line-too-long, invalid-name, missing-docstring, no-member + +from subprocess import run, PIPE +import sys +import unittest +from io import StringIO + + +class StreamTest(unittest.TestCase): + def setUp(self): + self.maxDiff = None + + def tearDown(self): + pass + + def catchio(self): + """ + Capture stdout as string. + """ + + self._saved_stdout = sys.stdout + self._strout = StringIO() + sys.stdout = self._strout + + def restoreio(self) -> str: + """ + Return captured output and restore stdout. + """ + + sys.stdout = self._saved_stdout + return self._strout.getvalue().strip() + + def teststreamer(self): + res = run(["gnssstreamer", "-h"], stdout=PIPE, check=False) + res = res.stdout.decode("utf-8") + self.assertEqual(res[0:19], "usage: gnssstreamer") + + def testserver(self): + res = run(["gnssserver", "-h"], stdout=PIPE, check=False) + res = res.stdout.decode("utf-8") + self.assertEqual(res[0:17], "usage: gnssserver") + + def testntripclient(self): + res = run(["gnssntripclient", "-h"], stdout=PIPE, check=False) + res = res.stdout.decode("utf-8") + self.assertEqual(res[0:22], "usage: gnssntripclient") + + def testsmqttlient(self): + res = run(["gnssmqttclient", "-h"], stdout=PIPE, check=False) + res = res.stdout.decode("utf-8") + self.assertEqual(res[0:21], "usage: gnssmqttclient") + + +if __name__ == "__main__": + # import sys;sys.argv = ['', 'Test.testName'] + unittest.main() diff --git a/tests/test_static.py b/tests/test_static.py index 70b50e5..3c299f9 100644 --- a/tests/test_static.py +++ b/tests/test_static.py @@ -23,6 +23,7 @@ format_conn, ipprot2int, ipprot2str, + format_dates, format_json, get_mp_distance, parse_config, @@ -53,6 +54,15 @@ def testitow2utc(self): res = str(itow2utc(387092000)) self.assertEqual(res, "11:31:14") + def testformatdates(self): + + http_date, server_date = format_dates() + self.assertRegex( + http_date, + r"\b[A-Za-z]{3}, \d{1,2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} UTC\b", + ) + self.assertRegex(server_date, r"\b\d{1,2} [A-Za-z]{3} \d{4}\b") + def testformatjson1(self): cls = "" json = (