rename update.sql, fix sorting bug, move conifg updates to pre_update…

… since pio command needs to know this config, and python scripts that import pioreactor need to know too

CHANGELOG.md

@@ -81,6 +81,7 @@
  - fixed typo `utils.local_persistant_storage` to `utils.local_persistent_storage`.
  - Kalman Filter database table is no longer populated. There is a way to re-add it, lmk.
  - moved intermittent cache location to `/tmp/pioreactor_cache/local_intermittent_pioreactor_metadata.sqlite`. This also determined by your configuration, see `[storage]`.
+- removed `pioreactor.utils.gpio_helpers`
  - removed `calibrations` export dataset. Use the export option on the /Calibrations page instead.
  - persistent storage is now on single sqlite3 database in `/home/pioreactor/.pioreactor/storage/local_persistent_pioreactor_metadata.sqlite`. This is configurable in your configuration.
  - When checking for calibrations in custom Dosing automations, users may have added:
@@ -96,12 +97,12 @@
             if "media_pump" not in cache:
             ...
     ```
-  - removed `pioreactor.utils.gpio_helpers`
 
 ### Bug fixes
  - Fix PWM3 not cleaning up correctly
  - Fixed Stirring not updating to best DC % when using a calibration after changing target RPM
  - Fixed a bug that could cause OD calibrations to map a small voltage value to a max OD.
+ - Fixed bug where dataset exports were not sorted correctly.
 
 ### 24.12.10
  - Hotfix for UI settings bug

pioreactor/actions/leader/export_experiment_data.py

@@ -100,7 +100,7 @@ def create_sql_query(
     table_or_subquery: str,
     existing_placeholders: dict[str, str],
     where_clauses: list[str] | None = None,
-    order_by: str | None = None,
+    order_by_col: str | None = None,
 ) -> tuple[str, dict[str, str]]:
     """
     Constructs an SQL query with SELECT, FROM, WHERE, and ORDER BY clauses.
@@ -113,9 +113,8 @@ def create_sql_query(
         query += f" WHERE {' AND '.join(where_clauses)}"
 
     # Add ORDER BY clause if provided
-    if order_by:
-        query += " ORDER BY :order_by"
-        existing_placeholders["order_by"] = order_by
+    if order_by_col:
+        query += f' ORDER BY "{order_by_col}"'
 
     return query, existing_placeholders
 
@@ -180,7 +179,7 @@ def export_experiment_data(
             filenames: list[str] = []
             placeholders: dict[str, str] = {}
 
-            order_by = dataset.default_order_by
+            order_by_col = dataset.default_order_by
             table_or_subquery = dataset.table or dataset.query
             assert table_or_subquery is not None
 
@@ -202,7 +201,7 @@ def export_experiment_data(
                 where_clauses.append(timespan_clause)
 
             query, placeholders = create_sql_query(
-                selects, table_or_subquery, placeholders, where_clauses, order_by
+                selects, table_or_subquery, placeholders, where_clauses, order_by_col
             )
             cursor.execute(query, placeholders)
 

pioreactor/actions/od_blank.py

@@ -172,6 +172,7 @@ def od_blank(
                     interval=1.5,
                     experiment=testing_experiment,  # use testing experiment to not pollute the database (and they would show up in the UI)
                     fake_data=whoami.is_testing_env(),
+                    calibration=True,
                 ) as od_stream:
                     # warm up OD reader
                     for count, _ in enumerate(od_stream, start=0):

pioreactor/background_jobs/od_reading.py

@@ -726,8 +726,6 @@ class ODReader(BackgroundJob):
     adc_reader: ADCReader
     ir_led_reference_tracker: IrLedReferenceTracker
     calibration_transformer:
-    unit:
-    experie
 
 
     Examples
@@ -1164,7 +1162,7 @@ def start_od_reading(
     fake_data: bool = False,
     unit: Optional[str] = None,
     experiment: Optional[str] = None,
-    calibration: structs.ODCalibration | None = None,
+    calibration: bool | structs.ODCalibration | None = None,
 ) -> ODReader:
     """
     This function prepares ODReader and other necessary transformation objects. It's a higher level API than using ODReader.
@@ -1204,7 +1202,11 @@ def start_od_reading(
         ir_led_reference_tracker = NullIrLedReferenceTracker()  # type: ignore
 
     # use an OD calibration?
-    if calibration:
+    if calibration is True:
+        calibration = load_active_calibration("od")
+        calibration_transformer = CachedCalibrationTransformer()
+        calibration_transformer.hydate_models(calibration)
+    elif isinstance(calibration, structs.ODCalibration):
         calibration_transformer = CachedCalibrationTransformer()
         calibration_transformer.hydate_models(calibration)
     else:

update_scripts/upcoming/pre_update.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -xeu
+
+
+export LC_ALL=C
+
+# Lower bound version
+min_version="24.12.5"
+
+# Get the current version of pio
+current_version=$(sudo -u pioreactor pio version)
+
+# Use sorting to determine if the current version is less than the minimum version
+is_valid=$(printf "%s\n%s" "$current_version" "$min_version" | sort -V | head -n1)
+
+# If the smallest version isn't the minimum version, then current version is too low
+if [ "$is_valid" != "$min_version" ]; then
+    sudo -u pioreactor pio log -l ERROR -m "Version error: installed version $current_version is lower than the minimum required version $min_version."
+    exit 1
+fi
+
+echo "Version check passed: $current_version"
+
+
+
+LEADER_HOSTNAME=$(crudini --get /home/pioreactor/.pioreactor/config.ini cluster.topology leader_hostname)
+
+
+# we need this config for downstream updates, so set it now.
+if [ "$HOSTNAME" = "$LEADER_HOSTNAME" ]; then
+
+    crudini  --set /home/pioreactor/.pioreactor/config.ini storage temporary_cache /tmp/pioreactor_cache/local_intermittent_pioreactor_metadata.sqlite \
+             --set /home/pioreactor/.pioreactor/config.ini storage persistent_cache /home/pioreactor/.pioreactor/storage/local_persistent_pioreactor_metadata.sqlite
+
+    sudo -u pioreactor pios sync-configs --shared || :
+
+fi

update_scripts/upcoming/update.sh

@@ -22,7 +22,10 @@ chown -R pioreactor:www-data $DB*
 chmod -R 770 $DB*
 
 # 2. make a calibration dir in all pioreactors
-sudo -u pioreactor mkdir -p "$STORAGE_DIR"/calibrations
+sudo -u pioreactor mkdir -p "$STORAGE_DIR"/calibrations/od
+sudo -u pioreactor mkdir -p "$STORAGE_DIR"/calibrations/media_pump
+sudo -u pioreactor mkdir -p "$STORAGE_DIR"/calibrations/waste_pump
+sudo -u pioreactor mkdir -p "$STORAGE_DIR"/calibrations/alt_media_pump
 
 # 3. install pyyaml (only leader has it, but workers need it now)
 sudo pip3 install "$SCRIPT_DIR"/PyYAML-6.0.2-cp311-cp311-linux_armv7l.whl