projecthorus · darksidelemm · Jul 4, 2021 · Jun 3, 2021 · Jun 3, 2021 · Jun 4, 2021
diff --git a/Dockerfile b/Dockerfile
@@ -6,8 +6,20 @@ FROM python:3.7-buster AS build
 # Upgrade base packages.
 RUN apt-get update && \
   apt-get upgrade -y && \
+  apt-get install -y \
+    cmake \
+    libusb-1.0-0-dev && \
   rm -rf /var/lib/apt/lists/*
 
+# Compile rtl-sdr from source.
+RUN git clone https://github.com/steve-m/librtlsdr.git /root/librtlsdr && \
+  mkdir -p /root/librtlsdr/build && \
+  cd /root/librtlsdr/build && \
+  cmake -DCMAKE_INSTALL_PREFIX=/root/target/usr/local -Wno-dev ../ && \
+  make && \
+  make install && \
+  rm -rf /root/librtlsdr
+
 # Copy in requirements.txt.
 COPY auto_rx/requirements.txt \
   /root/radiosonde_auto_rx/auto_rx/requirements.txt
@@ -41,13 +53,16 @@ RUN case $(uname -m) in \
   apt-get install -y \
   libatomic1 \
   rng-tools \
-  rtl-sdr \
   sox \
   tini \
   usbutils \
   ${extra_packages} && \
   rm -rf /var/lib/apt/lists/*
 
+# Copy rtl-sdr from the build container.
+COPY --from=build /root/target /
+RUN ldconfig
+
 # Copy any additional Python packages from the build container.
 COPY --from=build /root/.local /root/.local
 

diff --git a/Dockerfile.dev b/Dockerfile.dev
@@ -10,15 +10,26 @@ RUN case $(uname -m) in \
   apt-get update && \
   apt-get upgrade -y && \
   apt-get install -y \
+  cmake \
   libatomic1 \
+  libusb-1.0-0-dev \
   rng-tools \
-  rtl-sdr \
   sox \
   tini \
   usbutils \
   ${extra_packages} && \
   rm -rf /var/lib/apt/lists/*
 
+# Compile rtl-sdr from source and install.
+RUN git clone https://github.com/steve-m/librtlsdr.git /root/librtlsdr && \
+  mkdir -p /root/librtlsdr/build && \
+  cd /root/librtlsdr/build && \
+  cmake -Wno-dev ../ && \
+  make && \
+  make install && \
+  rm -rf /root/librtlsdr && \
+  ldconfig
+
 # Copy in requirements.txt.
 COPY auto_rx/requirements.txt \
   /tmp/requirements.txt

diff --git a/auto_rx/auto_rx.py b/auto_rx/auto_rx.py
@@ -245,9 +245,9 @@ def start_decoder(freq, sonde_type):
             timeout=config["rx_timeout"],
             telem_filter=telemetry_filter,
             rs92_ephemeris=rs92_ephemeris,
-            imet_location=config["station_code"],
             rs41_drift_tweak=config["rs41_drift_tweak"],
             experimental_decoder=config["experimental_decoders"][_exp_sonde_type],
+            save_raw_hex=config["save_raw_hex"]
         )
         autorx.sdr_list[_device_idx]["task"] = autorx.task_list[freq]["task"]
 
@@ -566,7 +566,9 @@ def telemetry_filter(telemetry):
     # This will need to be re-evaluated if we're still using this code in 2021!
     # UPDATE: Had some conf
8000
irmation that Vaisala will continue to use the alphanumeric numbering up until
     # ~2025-2030, so have expanded the regex to match (and also support some older RS92s)
-    vaisala_callsign_valid = re.match(r"[E-Z][0-5][\d][1-7]\d{4}", _serial)
+    # Modified 2021-06 to be more flexible and match older sondes, and reprogrammed sondes.
+    # Still needs a letter at the start, but the numbers don't need to match the format exactly.
+    vaisala_callsign_valid = re.match(r"[C-Z][\d][\d][\d]\d{4}", _serial)
 
     # Just make sure we're not getting the 'xxxxxxxx' unknown serial from the DFM decoder.
     if "DFM" in telemetry["type"]:

diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py
@@ -17,7 +17,7 @@
 # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus.
 # PATCH - Small changes, or minor feature additions.
 
-__version__ = "1.5.3"
+__version__ = "1.5.4"
 
 
 # Global Variables

diff --git a/auto_rx/autorx/config.py b/auto_rx/autorx/config.py
@@ -161,6 +161,7 @@ def read_auto_rx_config(filename, no_sdr_test=False):
         "save_detection_audio": False,
         "save_decode_audio": False,
         "save_decode_iq": False,
+        "save_raw_hex": False,
         # URL for the Habitat DB Server.
         # As of July 2018 we send via sondehub.org, which will allow us to eventually transition away
         # from using the habhub.org tracker, and leave it for use by High-Altitude Balloon Hobbyists.
@@ -553,6 +554,11 @@ def read_auto_rx_config(filename, no_sdr_test=False):
             auto_rx_config["sondehub_upload_rate"] = config.getint(
                 "sondehub", "sondehub_upload_rate"
             )
+            if auto_rx_config["sondehub_upload_rate"] < 10:
+                logging.warning(
+                    "Config - Clipped Sondehub update rate to lower limit of 10 seconds"
+                )
+                auto_rx_config["sondehub_upload_rate"] = 10
         except:
             logging.warning(
                 "Config - Did not find sondehub_enabled setting, using default (enabled / 15 seconds)."
@@ -614,6 +620,16 @@ def read_auto_rx_config(filename, no_sdr_test=False):
             )
             auto_rx_config["web_control"] = False
             auto_rx_config["web_password"] = "none"
+
+        try:
+            auto_rx_config["save_raw_hex"] = config.getboolean(
+                "debugging", "save_raw_hex"
+            )
+        except:
+            logging.warning(
+                "Config - Did not find save_raw_hex setting, using default (disabled)"
+            )
+            auto_rx_config["save_raw_hex"] = False
 
         # If we are being called as part of a unit test, just return the config now.
         if no_sdr_test:

diff --git a/auto_rx/autorx/decode.py b/auto_rx/autorx/decode.py
@@ -6,9 +6,11 @@
 #   Released under GNU GPL v3 or later
 #
 import autorx
+import datetime
 import logging
 import json
 import os
+import os.path
 import signal
 import subprocess
 import time
@@ -
F438
133,7 +135,7 @@ def __init__(
         rs92_ephemeris=None,
         rs41_drift_tweak=False,
         experimental_decoder=False,
-        imet_location="SONDE",
+        save_raw_hex=False
     ):
         """ Initialise and start a Sonde Decoder.
 
@@ -163,8 +165,7 @@ def __init__(
 
             rs41_drift_tweak (bool): If True, add a high-pass filter in the decode chain, which can improve decode performance on drifty SDRs.
             experimental_decoder (bool): If True, use the experimental fsk_demod-based decode chain.
-
-            imet_location (str): OPTIONAL - A location field which is use in the generation of iMet unique ID.
+            save_raw_hex (bool): If True, save the raw hex output from the decoder to a file.
         """
         # Thread running flag
         self.decoder_running = True
@@ -187,7 +188,16 @@ def __init__(
         self.rs92_ephemeris = rs92_ephemeris
         self.rs41_drift_tweak = rs41_drift_tweak
         self.experimental_decoder = experimental_decoder
-        self.imet_location = imet_location
+        self.save_raw_hex = save_raw_hex
+        self.raw_file = None
+
+        # Raw hex filename
+        if self.save_raw_hex:
+            _outfilename = f"{datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S')}_{self.sonde_type}_{int(self.sonde_freq)}.raw"
+            _outfilename = os.path.join(autorx.logging_path, _outfilename)
+            self.raw_file_option = "-r"
+        else:
+            self.raw_file_option = ""
 
         # iMet ID store. We latch in the first iMet ID we calculate, to avoid issues with iMet-1-RS units
         # which don't necessarily have a consistent packet count to time increment ratio.
@@ -281,6 +291,12 @@ def __init__(
             self.log_error("Could not generate decoder command. Not starting decoder.")
             self.decoder_running = False
         else:
+
+            if self.save_raw_hex:
+                self.log_debug(f"Opening {_outfilename} to save decoder raw data.")
+                # Open the log file in binary mode.
+                self.raw_file = open(_outfilename, 'wb')
+
             # Start up the decoder thread.
             self.decode_process = None
             self.async_reader = None
@@ -467,7 +483,7 @@ def generate_decoder_command(self):
                 decode_cmd += " tee decode_%s.wav |" % str(self.device_idx)
 
             # iMet-4 (IMET1RS) decoder
-            decode_cmd += "./imet1rs_dft --json 2>/dev/null"
+            decode_cmd += f"./imet1rs_dft --json {self.raw_file_option} 2>/dev/null"
 
         elif self.sonde_type == "IMET5":
             # iMet-54 Sondes
@@ -487,7 +503,7 @@ def generate_decoder_command(self):
 
             # iMet-54 Decoder
             decode_cmd += (
-                "./imet54mod --ecc --IQ 0.0 --lp - 48000 16 --json --ptu 2>/dev/null"
+                f"./imet54mod --ecc --IQ 0.0 --lp - 48000 16 --json --ptu 2>/dev/null"
             )
 
         elif self.sonde_type == "MRZ":
@@ -530,12 +546,12 @@ def generate_decoder_command(self):
             if self.save_decode_audio:
                 decode_cmd += " tee decode_%s.wav |" % str(self.device_idx)
 
-            # iMet-4 (IMET1RS) decoder
+            # LMS6-1680 decoder
             if self.inverted:
                 self.log_debug("Using inverted MK2A decoder.")
-                decode_cmd += "./mk2a_lms1680 -i --json 2>/dev/null"
+                decode_cmd += f"./mk2a_lms1680 -i --json {self.raw_file_option} 2>/dev/null"
             else:
-                decode_cmd += "./mk2a_lms1680 --json 2>/dev/null"
+                decode_cmd += f"./mk2a_lms1680 --json {self.raw_file_option} 2>/dev/null"
 
         elif self.sonde_type.startswith("LMS"):
             # LMS6 Decoder command.
@@ -588,7 +604,7 @@ def generate_decoder_command(self):
                 decode_cmd += " tee decode_%s.wav |" % str(self.device_idx)
 
             # Meisei IMS-100 decoder
-            decode_cmd += "./meisei100mod --json 2>/dev/null"
+            decode_cmd += f"./meisei100mod --json 2>/dev/null"
 
         elif self.sonde_type == "UDP":
             # UDP Input Mode.
@@ -655,7 +671,7 @@ def generate_decoder_command_experimental(self):
                 _baud_rate,
             )
 
-            decode_cmd = "./rs41mod --ptu2 --json --softin -i 2>/dev/null"
+            decode_cmd = f"./rs41mod --ptu2 --json --softin -i {self.raw_file_option} 2>/dev/null"
 
             # RS41s transmit pulsed beacons - average over the last 2 frames, and use a peak-hold
             demod_stats = FSKDemodStats(averaging_time=2.0, peak_hold=True)
@@ -775,7 +791,7 @@ def generate_decoder_command_experimental(self):
 
             # DFM decoder
             decode_cmd = (
-                "./dfm09mod -vv --ecc --json --dist --auto --softin -i 2>/dev/null"
+                f"./dfm09mod -vv --ecc --json --dist --auto --softin -i {self.raw_file_option.upper()} 2>/dev/null"
             )
 
             # DFM sondes transmit continuously - average over the last 2 frames, and peak hold
@@ -815,7 +831,7 @@ def generate_decoder_command_experimental(self):
             )
 
             # M10 decoder
-            decode_cmd = "./m10mod --json --ptu -vvv --softin -i 2>/dev/null"
+            decode_cmd = f"./m10mod --json --ptu -vvv --softin -i {self.raw_file_option} 2>/dev/null"
 
             # M10 sondes transmit in short, irregular pulses - average over the last 2 frames, and use a peak hold
             demod_stats = FSKDemodStats(averaging_time=2.0, peak_hold=True)
@@ -853,7 +869,7 @@ def generate_decoder_command_experimental(self):
             )
 
             # M20 decoder
-            decode_cmd = "./mXXmod --json --ptu -vvv --softin -i 2>/dev/null"
+            decode_cmd = f"./mXXmod --json --ptu -vvv --softin -i {self.raw_file_option} 2>/dev/null"
 
             # M20 sondes transmit in short, irregular pulses - average over the last 2 frames, and use a peak hold
             demod_stats = FSKDemodStats(averaging_time=2.0, peak_hold=True)
@@ -892,7 +908,7 @@ def generate_decoder_command_experimental(self):
                 _baud_rate,
             )
 
-            decode_cmd = "./lms6Xmod --json --softin --vit2 -i 2>/dev/null"
+            decode_cmd = f"./lms6Xmod --json --softin --vit2 -i {self.raw_file_option} 2>/dev/null"
 
             # LMS sondes transmit continuously - average over the last 2 frames, and use a peak hold
             demod_stats = FSKDemodStats(averaging_time=2.0, peak_hold=True)
@@ -931,7 +947,7 @@ def generate_decoder_command_experimental(self):
                 _baud_rate,
             )
 
-            decode_cmd = "./imet54mod --ecc --json --softin -i --ptu 2>/dev/null"
+            decode_cmd = f"./imet54mod --ecc --json --softin -i --ptu 2>/dev/null"
 
             # iMet54 sondes transmit in bursts. Use a peak hold.
             demod_stats = FSKDemodStats(averaging_time=2.0, peak_hold=True)
@@ -972,7 +988,7 @@ def generate_decoder_command_experimental(self):
             )
 
             # MRZ decoder
-            decode_cmd = "./mp3h1mod --auto --json --softin --ptu 2>/dev/null"
+            decode_cmd = f"./mp3h1mod --auto --json --softin --ptu 2>/dev/null"
 
             # MRZ sondes transmit continuously - average over the last frame, and use a peak hold
             demod_stats = FSKDemodStats(averaging_time=1.0, peak_hold=True)
@@ -1126,9 +1142,14 @@ def handle_decoder_line(self, data):
 
         # Don't even try and decode lines which don't start with a '{'
         # These may be other output from the decoder, which we shouldn't try to parse.
-        # TODO: Perhaps we should add the option to log the raw data output from the decoders?
+        # If we have raw logging enabled, log these lines to disk.
         if data.decode("ascii")[0] != "{":
-            return
+
+            # Save the line verbatim to the raw data file, if we have that enabled
+            if self.raw_file:
+                self.raw_file.write(data)
+            else:
+                return
 
         else:
             try:
@@ -1255,14 +1276,13 @@ def handle_decoder_line(self, data):
                 # Generate a unique ID based on the power-on time and frequency, as iMet sondes don't send one.
                 # Latch this ID and re-use it for the entire decode run.
                 if self.imet_id == None:
-                    self.imet_id = imet_unique_id(_telemetry, custom=self.imet_location)
+                    self.imet_id = imet_unique_id(_telemetry)
 
                 # Re-generate the datetime string.
                 _telemetry["datetime"] = _telemetry["datetime_dt"].strftime(
                     "%Y-%m-%dT%H:%M:%SZ"
                 )
                 _telemetry["id"] = self.imet_id
-                _telemetry["station_code"] = self.imet_location
 
             # iMet-54 Specific Actions
             if self.sonde_type == "IMET5":
@@ -1377,6 +1397,9 @@ def stop(self, nowait=False):
 
         if self.decoder is not None and (not nowait):
             self.decoder.join()
+
+        if self.raw_file:
+            self.raw_file.close()
 
     def running(self):
         """ Check if the decoder subprocess is running. 

diff --git a/auto_rx/autorx/gps.py b/auto_rx/autorx/gps.py
@@ -78,7 +78,7 @@ def get_almanac(destination="almanac.txt", timeout=20):
             logging.error("GPS Grabber - Downloaded file is not a GPS almanac.")
             return None
     except Exception as e:
-        logging.error("GPS Grabber - Failed to download almanac data - " % str(e))
+        logging.error(f"GPS Grabber - Failed to download almanac data - {str(e)}")
         return None
 
 

diff --git a/auto_rx/autorx/log_files.py b/auto_rx/autorx/log_files.py
@@ -20,6 +20,7 @@
 from dateutil.parser import parse
 from autorx.utils import (
     short_type_lookup,
+    short_short_type_lookup,
     readable_timedelta,
     strip_sonde_serial,
     position_info,
@@ -67,6 +68,7 @@ def log_filename_to_stats(filename, quicklook=False):
         # Third field is the sonde type, in 'shortform'
         _type = _fields[2]
         _type_str = short_type_lookup(_type)
+        _short_type = short_short_type_lookup(_type)
 
         # Fourth field is the sonde frequency in kHz
         _freq = float(_fields[3]) / 1e3
@@ -76,6 +78,7 @@ def log_filename_to_stats(filename, quicklook=False):
             "age": _time_delta,
             "serial": _serial,
             "type": _type_str,
+            "short_type": _short_type,
             "freq": _freq,
             "lines": _lines,
         }
@@ -87,6 +90,8 @@ def log_filename_to_stats(filename, quicklook=False):
                     _output["first"] = _quick["first"]
                     _output["last"] = _quick["last"]
                     _output["has_snr"] = _quick["has_snr"]
+                    _output["max_range"] = int(max(_output["first"]["range_km"],_output["last"]["range_km"]))
+                    _output["min_height"] = int(_output["last"]["alt"])
             except Exception as e:
                 logging.error(f"Could not quicklook file {filename}: {str(e)}")
 

diff --git a/auto_rx/autorx/sonde_specific.py b/auto_rx/autorx/sonde_specific.py
@@ -51,7 +51,7 @@ def fix_datetime(datetime_str, local_dt_str=None):
 #
 
 
-def imet_unique_id(telemetry, custom=""):
+def imet_unique_id(telemetry, custom="SONDE"):
     """
 	Generate a 'unique' imet radiosonde ID based on the power-on time, frequency, and an optional location code.
 	This requires the following fields be present in the telemetry dict: