TIL that PRAGMAs are per connection

CHANGELOG.md

@@ -72,6 +72,8 @@
  - new SQL table for `historical_experiment_assignments` that stores historical assignments to experiments.
  - UI performance improvements
  - Better terminal plots
+ - Customs charts in the UI are now downsampled like the other charts.
+ - More logging in experiment profiles
 
 ### Breaking changes
  - `use_calibration` under `od_reading.config` is deprecated. Use the calibrations "active" state instead.
@@ -103,6 +105,8 @@
  - Fixed Stirring not updating to best DC % when using a calibration after changing target RPM
  - Fixed a bug that could cause OD calibrations to map a small voltage value to a max OD.
  - Fixed bug where dataset exports were not sorted correctly.
+ - em-dashes are now replaced in config.ini on save.
+ - Fixed a bug where errors on the Experiment Profiles page weren't properly displayed.
 
 ### 24.12.10
  - Hotfix for UI settings bug

config.dev.ini

@@ -66,7 +66,7 @@ smoothing_penalizer=700.0
 
 [storage]
 database=.pioreactor/storage/pioreactor.sqlite
-temporary_cache=local_intermittent_pioreactor_metadata.sqlite
+temporary_cache=/tmp/pioreactor_cache/local_intermittent_pioreactor_metadata.sqlite
 persistent_cache=.pioreactor/storage/local_persistent_pioreactor_metadata.sqlite
 
 

pioreactor/actions/leader/experiment_profile.py

@@ -539,6 +539,7 @@ def _callable() -> None:
             if dry_run:
                 logger.info(f"Dry-run: Starting {job_name} on {unit} with options {options} and args {args}.")
             else:
+                logger.debug(f"Starting {job_name} on {unit} with options {options} and args {args}.")
                 patch_into_leader(
                     f"/api/workers/{unit}/jobs/run/job_name/{job_name}/experiments/{experiment}",
                     json={
@@ -579,6 +580,7 @@ def _callable() -> None:
             if dry_run:
                 logger.info(f"Dry-run: Pausing {job_name} on {unit}.")
             else:
+                logger.debug(f"Pausing {job_name} on {unit}.")
                 patch_into_leader(
                     f"/api/workers/{unit}/jobs/update/job_name/{job_name}/experiments/{experiment}",
                     json={"settings": {"$state": "sleeping"}},
@@ -616,6 +618,7 @@ def _callable() -> None:
             if dry_run:
                 logger.info(f"Dry-run: Resuming {job_name} on {unit}.")
             else:
+                logger.debug(f"Resuming {job_name} on {unit}.")
                 patch_into_leader(
                     f"/api/workers/{unit}/jobs/update/job_name/{job_name}/experiments/{experiment}",
                     json={"settings": {"$state": "ready"}},
@@ -653,6 +656,7 @@ def _callable() -> None:
             if dry_run:
                 logger.info(f"Dry-run: Stopping {job_name} on {unit}.")
             else:
+                logger.debug(f"Stopping {job_name} on {unit}.")
                 patch_into_leader(
                     f"/api/workers/{unit}/jobs/stop/job_name/{job_name}/experiments/{experiment}",
                 )
@@ -693,6 +697,7 @@ def _callable() -> None:
 
             else:
                 for setting, value in evaluate_options(options, env).items():
+                    logger.debug(f"Updating {setting} to {value} in {job_name} on {unit}.")
                     patch_into_leader(
                         f"/api/workers/{unit}/jobs/update/job_name/{job_name}/experiments/{experiment}",
                         json={"settings": {setting: value}},

pioreactor/actions/leader/export_experiment_data.py

@@ -162,6 +162,18 @@ def export_experiment_data(
         con.set_trace_callback(logger.debug)
 
         cursor = con.cursor()
+        cursor.executescript(
+            """
+            PRAGMA journal_mode=WAL;
+            PRAGMA synchronous = 1; -- aka NORMAL, recommended when using WAL
+            PRAGMA temp_store = 2;  -- stop writing small files to disk, use mem
+            PRAGMA busy_timeout = 15000;
+            PRAGMA foreign_keys = ON;
+            PRAGMA synchronous = NORMAL;
+            PRAGMA auto_vacuum = INCREMENTAL;
+            PRAGMA cache_size = -20000;
+        """
+        )
 
         for dataset_name in dataset_names:
             try:

pioreactor/calibrations/stirring_calibration.py

@@ -18,7 +18,6 @@
 from pioreactor.utils import managed_lifecycle
 from pioreactor.utils.math_helpers import simple_linear_regression
 from pioreactor.utils.timing import current_utc_datetime
-from pioreactor.whoami import get_assigned_experiment_name
 from pioreactor.whoami import get_testing_experiment_name
 from pioreactor.whoami import get_unit_name
 
@@ -40,7 +39,7 @@ def run_stirring_calibration(
     action_name = "stirring_calibration"
     logger = create_logger(action_name)
 
-    with managed_lifecycle(unit, get_assigned_experiment_name(unit), action_name) as lc:
+    with managed_lifecycle(unit, experiment, action_name) as lc:
         logger.info("Starting stirring calibration.")
 
         if is_pio_job_running("stirring"):

pioreactor/tests/test_update_scripts.py

@@ -5,6 +5,14 @@
 from typing import Generator
 
 
+def find_sql_scripts(directory: str) -> Generator[str, None, None]:
+    """Recursively find all SQL script files in the specified directory."""
+    for root, dirs, files in os.walk(directory):
+        for file in files:
+            if file.endswith(".sql"):
+                yield os.path.join(root, file)
+
+
 def find_shell_scripts(directory: str) -> Generator[str, None, None]:
     """Recursively find all shell script files in the specified directory."""
     types = {"update.sh", "pre_update.sh", "post_update.sh"}
@@ -34,6 +42,20 @@ def test_pio_commands() -> None:
     assert not error_msgs, "\n".join(error_msgs)
 
 
+def test_sql_scripts_start_with_our_PRAGMA() -> None:
+    script_directory = "update_scripts/upcoming"
+    scripts = find_sql_scripts(script_directory)
+    error_msgs = []
+
+    for script in scripts:
+        with open(script, "r") as file:
+            first_line = file.readline().strip()
+            if not first_line.startswith("PRAGMA"):
+                error_msgs.append(f"Error in {script}: SQL scripts must start with a PRAGMA statement.")
+
+    assert not error_msgs, "\n".join(error_msgs)
+
+
 def test_no_restarting_huey_service() -> None:
     # this can mess with updating if we interrupt huey.
     script_directory = "update_scripts"

pioreactor/utils/__init__.py

@@ -292,7 +292,18 @@ def __enter__(self):
 
         self.conn = sqlite3.connect(self.db_path, isolation_level=None, detect_types=sqlite3.PARSE_DECLTYPES)
         self.cursor = self.conn.cursor()
-        self.cursor.execute("PRAGMA journal_mode=WAL;")
+        self.cursor.executescript(
+            """
+            PRAGMA journal_mode=WAL;
+            PRAGMA synchronous = 1; -- aka NORMAL, recommended when using WAL
+            PRAGMA temp_store = 2;  -- stop writing small files to disk, use mem
+            PRAGMA busy_timeout = 15000;
+            PRAGMA foreign_keys = ON;
+            PRAGMA synchronous = NORMAL;
+            PRAGMA auto_vacuum = INCREMENTAL;
+            PRAGMA cache_size = -20000;
+        """
+        )
         self._initialize_table()
         return self
 
@@ -603,12 +614,21 @@ def __init__(self) -> None:
         db_path = config.get("storage", "temporary_cache")
         self.conn = sqlite3.connect(db_path, isolation_level=None)
         self.cursor = self.conn.cursor()
-        self.cursor.execute("PRAGMA journal_mode=WAL;")
         self._create_tables()
 
     def _create_tables(self) -> None:
         # TODO: add a created_at, updated_at to pio_job_published_settings
         create_table_query = """
+
+        PRAGMA journal_mode=WAL;
+        PRAGMA synchronous = 1; -- aka NORMAL, recommended when using WAL
+        PRAGMA temp_store = 2;  -- stop writing small files to disk, use mem
+        PRAGMA busy_timeout = 15000;
+        PRAGMA foreign_keys = ON;
+        PRAGMA synchronous = NORMAL;
+        PRAGMA auto_vacuum = INCREMENTAL;
+        PRAGMA cache_size = -20000;
+
         CREATE TABLE IF NOT EXISTS pio_job_metadata (
             id           INTEGER PRIMARY KEY AUTOINCREMENT,
             unit         TEXT NOT NULL,

pioreactor/utils/sqlite_worker.py

@@ -59,6 +59,18 @@ def __init__(self, file_name: str, max_queue_size: int = 100, raise_on_error: bo
             file_name, check_same_thread=False, detect_types=sqlite3.PARSE_DECLTYPES
         )
         self._sqlite3_cursor = self._sqlite3_conn.cursor()
+        self._sqlite3_cursor.executescript(
+            """
+            PRAGMA journal_mode=WAL;
+            PRAGMA synchronous = 1; -- aka NORMAL, recommended when using WAL
+            PRAGMA temp_store = 2;  -- stop writing small files to disk, use mem
+            PRAGMA busy_timeout = 15000;
+            PRAGMA foreign_keys = ON;
+            PRAGMA synchronous = NORMAL;
+            PRAGMA auto_vacuum = INCREMENTAL;
+            PRAGMA cache_size = -20000;
+        """
+        )
         self._sql_queue: Queue[tuple[str, str, tuple]] = Queue(maxsize=max_queue_size)
         self._results: dict[str, list | str] = {}
         self._max_queue_size = max_queue_size

update_scripts/upcoming/update.sh

@@ -34,7 +34,7 @@ sudo bash /usr/local/bin/create_diskcache.sh
 # 5. replace old calibrations with new yaml files. This doesn't delete old calibrations
 sudo -u pioreactor python "$SCRIPT_DIR"/cal_convert.py "$STORAGE_DIR"/od_calibrations/cache.db
 sudo -u pioreactor python "$SCRIPT_DIR"/cal_convert.py "$STORAGE_DIR"/pump_calibrations/cache.db
-chown -R pioreactor:pioreactor "$STORAGE_DIR"/calibrations/
+chown -R pioreactor:www-data "$STORAGE_DIR"/calibrations/
 
 sudo -u pioreactor python "$SCRIPT_DIR"/cal_active.py "$STORAGE_DIR"/current_pump_calibrations/cache.db
 sudo -u pioreactor python "$SCRIPT_DIR"/cal_active.py "$STORAGE_DIR"/current_od_calibrations/cache.db
@@ -48,6 +48,9 @@ if [ "$HOSTNAME" = "$LEADER_HOSTNAME" ]; then
     # 7. fix any bad pioreactor start up systemd services
     rm -f  /usr/lib/systemd/system/pioreactor_startup_run@.service
     cp  "$SCRIPT_DIR"/pioreactor_startup_run@.service /etc/systemd/system/
+
+
+    # 8. add yaml mimetype
     echo "application/yaml               yaml yml" | sudo tee -a /etc/mime.types
 
 fi

update_scripts/upcoming/update.sql

@@ -1,3 +1,11 @@
+PRAGMA journal_mode=WAL;
+PRAGMA synchronous = 1; -- aka NORMAL, recommended when using WAL
+PRAGMA temp_store = 2;  -- stop writing small files to disk, use mem
+PRAGMA busy_timeout = 15000;
+PRAGMA foreign_keys = ON;
+PRAGMA synchronous = NORMAL;
+PRAGMA auto_vacuum = INCREMENTAL;
+PRAGMA cache_size = -20000;
 
 --- see triggers for how this is populated!
 CREATE TABLE IF NOT EXISTS experiment_worker_assignments_history (