intermediate backup

optimizer/forecasting/base.py

@@ -0,0 +1,22 @@
+# forecasting/base.py
+from typing import List, Dict, Any
+import pandas as pd
+import numpy as np
+
+
+class ForecastProvider:
+    def get_forecasts(self, 
+                    historical_data: pd.DataFrame,
+                    forecast_horizons: List[int],
+                    optimization_horizon: int) -> Dict[int, np.ndarray]:
+        """Returns forecasts for each requested horizon."""
+        pass
+
+    def get_required_lookback(self) -> int:
+        """Returns the minimum number of historical data points required."""
+        pass
+
+    def get_forecast_horizons(self) -> List[int]:
+        """Returns the list of forecast horizons."""
+        pass
+

optimizer/forecasting/ensemble.py

@@ -0,0 +1,188 @@
+import logging
+from typing import List, Dict, Any, Optional
+
+import numpy as np
+import pandas as pd
+import torch
+from sklearn.preprocessing import StandardScaler, MinMaxScaler
+
+from .base import ForecastProvider
+from forecasting_model.utils import FeatureConfig
+from forecasting_model.train.model import LSTMForecastLightningModule
+from forecasting_model import engineer_features
+from optimizer.forecasting.utils import interpolate_forecast
+
+logger = logging.getLogger(__name__)
+
+
+
+class EnsembleProvider(ForecastProvider):
+    """Provides forecasts using an ensemble of trained LSTM models."""
+
+    def __init__(
+        self,
+        fold_artifacts: List[Dict[str, Any]],
+        ensemble_method: str,
+        ensemble_feature_config: FeatureConfig, # Assumed consistent across folds by loading logic
+        ensemble_target_col: str, # Assumed consistent
+    ):
+        if not fold_artifacts:
+            raise ValueError("EnsembleProvider requires at least one fold artifact.")
+
+        self.fold_artifacts = fold_artifacts
+        self.ensemble_method = ensemble_method
+        # Store common config for reference, but use fold-specific details in get_forecast
+        self.ensemble_feature_config = ensemble_feature_config
+        self.ensemble_target_col = ensemble_target_col
+        self.common_forecast_horizons = sorted(ensemble_feature_config.forecast_horizon) # Assumed consistent
+
+        # Calculate max lookback needed across all folds
+        max_lookback = 0
+        for i, fold in enumerate(fold_artifacts):
+            try:
+                fold_feature_config = fold['feature_config']
+                fold_seq_len = fold_feature_config.sequence_length
+
+                feature_lookback = 0
+                if fold_feature_config.lags:
+                    feature_lookback = max(feature_lookback, max(fold_feature_config.lags))
+                if fold_feature_config.rolling_window_sizes:
+                     feature_lookback = max(feature_lookback, max(w - 1 for w in fold_feature_config.rolling_window_sizes))
+
+                fold_total_lookback = fold_seq_len + feature_lookback
+                max_lookback = max(max_lookback, fold_total_lookback)
+            except KeyError as e:
+                 raise ValueError(f"Fold artifact {i} is missing expected key: {e}") from e
+            except Exception as e:
+                 raise ValueError(f"Error processing fold artifact {i} for lookback calculation: {e}") from e
+
+        self._required_lookback = max_lookback
+        logger.debug(f"EnsembleProvider initialized with {len(fold_artifacts)} folds. Method: '{ensemble_method}'. Required lookback: {self._required_lookback}")
+
+        if ensemble_method not in ['mean', 'median']:
+             raise ValueError(f"Unsupported ensemble method: {ensemble_method}. Use 'mean' or 'median'.")
+
+    def get_required_lookback(self) -> int:
+        return self._required_lookback
+
+    def get_forecast(
+        self,
+        historical_data_slice: pd.DataFrame,
+        optimization_horizon_hours: int
+    ) -> np.ndarray | None:
+        """
+        Generates forecasts from each fold model, interpolates, and aggregates.
+        """
+        logger.debug(f"EnsembleProvider: Generating forecast for {optimization_horizon_hours} hours using {self.ensemble_method}.")
+        if len(historical_data_slice) < self._required_lookback:
+             logger.error(f"Insufficient historical data provided. Need {self._required_lookback}, got {len(historical_data_slice)}.")
+             return None
+
+        fold_forecasts_interpolated = []
+        last_actual_price = historical_data_slice[self.ensemble_target_col].iloc[-1] # Common anchor for all folds
+
+        for i, fold_artifact in enumerate(self.fold_artifacts):
+            fold_id = fold_artifact.get("fold_id", i + 1)
+            try:
+                fold_model: LSTMForecastLightningModule = fold_artifact['model_instance']
+                fold_feature_config: FeatureConfig = fold_artifact['feature_config']
+                fold_target_scaler: Optional[Any] = fold_artifact['target_scaler']
+                fold_target_col: str = fold_artifact['main_forecasting_config'].data.target_col # Use fold specific target
+                fold_seq_len = fold_feature_config.sequence_length
+                fold_horizons = sorted(fold_feature_config.forecast_horizon)
+
+                # Calculate lookback needed *for this specific fold* to check slice length
+                fold_feature_lookback = 0
+                if fold_feature_config.lags: fold_feature_lookback = max(fold_feature_lookback, max(fold_feature_config.lags))
+                if fold_feature_config.rolling_window_sizes: fold_feature_lookback = max(fold_feature_lookback, max(w - 1 for w in fold_feature_config.rolling_window_sizes))
+                fold_total_lookback = fold_seq_len + fold_feature_lookback
+
+                if len(historical_data_slice) < fold_total_lookback:
+                    logger.warning(f"Fold {fold_id}: Skipping fold. Insufficient historical data in slice for this fold's lookback ({fold_total_lookback} needed).")
+                    continue
+
+                # 1. Feature Engineering (using fold's config)
+                # Slice needs to be long enough for this fold's total lookback.
+                # The input slice `historical_data_slice` should already be long enough based on max_lookback.
+                engineered_df_fold = engineer_features(historical_data_slice.copy(), fold_target_col, fold_feature_config)
+
+                if engineered_df_fold.isnull().any().any():
+                    logger.warning(f"Fold {fold_id}: NaNs found after feature engineering. Attempting fill.")
+                    engineered_df_fold = engineered_df_fold.ffill().bfill()
+                    if engineered_df_fold.isnull().any().any():
+                        logger.error(f"Fold {fold_id}: NaNs persist after fill. Skipping fold.")
+                        continue
+
+                # 2. Create *one* input sequence (using fold's sequence length)
+                if len(engineered_df_fold) < fold_seq_len:
+                    logger.error(f"Fold {fold_id}: Engineered data ({len(engineered_df_fold)}) is shorter than fold sequence length ({fold_seq_len}). Skipping fold.")
+                    continue
+
+                input_sequence_data_fold = engineered_df_fold.iloc[-fold_seq_len:].copy()
+                feature_columns_fold = [col for col in engineered_df_fold.columns if col != fold_target_col] # Example
+                if not feature_columns_fold: feature_columns_fold = engineered_df_fold.columns.tolist()
+                input_sequence_np_fold = input_sequence_data_fold[feature_columns_fold].values
+
+                if input_sequence_np_fold.shape != (fold_seq_len, len(feature_columns_fold)):
+                     logger.error(f"Fold {fold_id}: Input sequence has wrong shape. Expected ({fold_seq_len}, {len(feature_columns_fold)}), got {input_sequence_np_fold.shape}. Skipping fold.")
+                     continue
+
+                input_tensor_fold = torch.FloatTensor(input_sequence_np_fold).unsqueeze(0)
+
+                # 3. Run Inference (using fold's model)
+                fold_model.eval()
+                with torch.no_grad():
+                    predictions_scaled_fold = fold_model(input_tensor_fold) # Shape (1, num_fold_horizons)
+
+                if predictions_scaled_fold.ndim != 2 or predictions_scaled_fold.shape[0] != 1 or predictions_scaled_fold.shape[1] != len(fold_horizons):
+                     logger.error(f"Fold {fold_id}: Prediction output shape mismatch. Expected (1, {len(fold_horizons)}), got {predictions_scaled_fold.shape}. Skipping fold.")
+                     continue
+
+                predictions_scaled_np_fold = predictions_scaled_fold.squeeze(0).cpu().numpy()
+
+                # 4. Inverse Transform (using fold's scaler)
+                predictions_original_scale_fold = predictions_scaled_np_fold
+                if fold_target_scaler:
+                    try:
+                        predictions_original_scale_fold = fold_target_scaler.inverse_transform(predictions_scaled_np_fold.reshape(-1, 1)).flatten()
+                    except Exception as e:
+                        logger.error(f"Fold {fold_id}: Failed to apply inverse transform: {e}. Skipping fold.", exc_info=True)
+                        continue
+
+                # 5. Interpolate (using fold's horizons)
+                interpolated_forecast_fold = interpolate_forecast(
+                    native_horizons=fold_horizons,
+                    native_predictions=predictions_original_scale_fold,
+                    target_horizon=optimization_horizon_hours,
+                    last_known_actual=last_actual_price
+                )
+
+                if interpolated_forecast_fold is not None:
+                     fold_forecasts_interpolated.append(interpolated_forecast_fold)
+                     logger.debug(f"Fold {fold_id}: Successfully generated interpolated forecast.")
+                else:
+                     logger.warning(f"Fold {fold_id}: Interpolation failed. Skipping fold.")
+
+            except Exception as e:
+                logger.error(f"Error processing ensemble fold {fold_id}: {e}", exc_info=True)
+                continue # Skip this fold on error
+
+        # --- Aggregation ---
+        if not fold_forecasts_interpolated:
+            logger.error("No successful forecasts generated from any ensemble folds.")
+            return None
+
+        logger.debug(f"Aggregating forecasts from {len(fold_forecasts_interpolated)} folds using '{self.ensemble_method}'.")
+        stacked_predictions = np.stack(fold_forecasts_interpolated, axis=0) # Shape (n_folds, target_horizon)
+
+        if self.ensemble_method == 'mean':
+            final_ensemble_forecast = np.mean(stacked_predictions, axis=0)
+        elif self.ensemble_method == 'median':
+            final_ensemble_forecast = np.median(stacked_predictions, axis=0)
+        else:
+            # Should be caught in __init__, but double-check
+            logger.error(f"Internal error: Invalid ensemble method '{self.ensemble_method}' during aggregation.")
+            return None
+
+        logger.debug(f"EnsembleProvider: Successfully generated forecast.")
+        return final_ensemble_forecast

optimizer/forecasting/single_model.py

@@ -0,0 +1,150 @@
+import logging
+from typing import List, Dict, Any, Optional
+
+import numpy as np
+import pandas as pd
+import torch
+from sklearn.preprocessing import StandardScaler, MinMaxScaler
+
+# Imports from our project structure
+from .base import ForecastProvider
+from forecasting_model.utils import FeatureConfig
+from forecasting_model.train.model import LSTMForecastLightningModule
+from forecasting_model import engineer_features
+from optimizer.forecasting.utils import interpolate_forecast
+
+logger = logging.getLogger(__name__)
+
+
+class SingleModelProvider(ForecastProvider):
+    """Provides forecasts using a single trained LSTM model."""
+
+    def __init__(
+        self,
+        model_instance: LSTMForecastLightningModule,
+        feature_config: FeatureConfig,
+        target_col: str,
+        target_scaler: Optional[Any], # BaseEstimator, TransformerMixin -> more specific if possible
+        # input_size: int # Not needed directly if model instance is configured
+    ):
+        self.model = model_instance
+        self.feature_config = feature_config
+        self.target_col = target_col
+        self.target_scaler = target_scaler
+        self.sequence_length = feature_config.sequence_length
+        self.forecast_horizons = sorted(feature_config.forecast_horizon) # Ensure sorted
+
+        # Calculate required lookback for feature engineering
+        feature_lookback = 0
+        if feature_config.lags:
+            feature_lookback = max(feature_lookback, max(feature_config.lags))
+        if feature_config.rolling_window_sizes:
+            # Rolling window of size W needs W-1 previous points
+            feature_lookback = max(feature_lookback, max(w - 1 for w in feature_config.rolling_window_sizes))
+
+        # Total lookback: sequence length for model input + feature engineering needs
+        # We need `sequence_length` points for the *last* input sequence.
+        # The first point of that sequence needs `feature_lookback` points before it.
+        # So, total points needed before the *end* of the input sequence is sequence_length + feature_lookback.
+        # Since the input sequence ends *before* the first forecast point (t=1),
+        # we need `sequence_length + feature_lookback` points before t=1.
+        self._required_lookback = self.sequence_length + feature_lookback
+        logger.debug(f"SingleModelProvider initialized. Required lookback: {self._required_lookback} (SeqLen: {self.sequence_length}, FeatLookback: {feature_lookback})")
+
+
+    def get_required_lookback(self) -> int:
+        return self._required_lookback
+
+    def get_forecast(
+        self,
+        historical_data_slice: pd.DataFrame,
+        optimization_horizon_hours: int
+    ) -> np.ndarray | None:
+        """
+        Generates forecast using the single model and interpolates to hourly resolution.
+        """
+        logger.debug(f"SingleModelProvider: Generating forecast for {optimization_horizon_hours} hours.")
+        if len(historical_data_slice) < self._required_lookback:
+             logger.error(f"Insufficient historical data provided. Need {self._required_lookback}, got {len(historical_data_slice)}.")
+             return None
+
+        try:
+            # 1. Feature Engineering
+            # Use the provided slice which already includes the lookback.
+            engineered_df = engineer_features(historical_data_slice.copy(), self.target_col, self.feature_config)
+
+            # Check for NaNs after feature engineering before creating sequences
+            if engineered_df.isnull().any().any():
+                 logger.warning("NaNs found after feature engineering. Attempting to fill with ffill/bfill.")
+                 # Be careful about filling target vs features if needed
+                 engineered_df = engineered_df.ffill().bfill()
+                 if engineered_df.isnull().any().any():
+                     logger.error("NaNs persist after fill. Cannot create sequences.")
+                     return None
+
+            # 2. Create *one* input sequence ending at the last point of the historical slice
+            # This sequence is used to predict starting from the next hour (t=1)
+            if len(engineered_df) < self.sequence_length:
+                 logger.error(f"Engineered data ({len(engineered_df)}) is shorter than sequence length ({self.sequence_length}).")
+                 return None
+
+            input_sequence_data = engineered_df.iloc[-self.sequence_length:].copy()
+
+            # Convert sequence data to numpy array (excluding target if model expects it that way)
+            # Assuming model takes all engineered features as input
+            # TODO: Verify the exact features the model expects (target included/excluded?)
+            # Assuming all columns except maybe the original target are features
+            feature_columns = [col for col in engineered_df.columns if col != self.target_col] # Example
+            if not feature_columns: feature_columns = engineered_df.columns.tolist() # Use all if target wasn't dropped
+            input_sequence_np = input_sequence_data[feature_columns].values
+
+            if input_sequence_np.shape != (self.sequence_length, len(feature_columns)):
+                logger.error(f"Input sequence has wrong shape. Expected ({self.sequence_length}, {len(feature_columns)}), got {input_sequence_np.shape}")
+                return None
+
+            input_tensor = torch.FloatTensor(input_sequence_np).unsqueeze(0) # Add batch dim
+
+            # 3. Run Inference
+            self.model.eval()
+            with torch.no_grad():
+                # Model output shape: (1, num_horizons)
+                predictions_scaled = self.model(input_tensor)
+
+            if predictions_scaled.ndim != 2 or predictions_scaled.shape[0] != 1 or predictions_scaled.shape[1] != len(self.forecast_horizons):
+                logger.error(f"Model prediction output shape mismatch. Expected (1, {len(self.forecast_horizons)}), got {predictions_scaled.shape}.")
+                return None
+
+            predictions_scaled_np = predictions_scaled.squeeze(0).cpu().numpy() # Shape: (num_horizons,)
+
+            # 4. Inverse Transform
+            predictions_original_scale = predictions_scaled_np
+            if self.target_scaler:
+                try:
+                    # Scaler expects shape (n_samples, n_features), even if n_features=1
+                    predictions_original_scale = self.target_scaler.inverse_transform(predictions_scaled_np.reshape(-1, 1)).flatten()
+                    logger.debug("Applied inverse transform to predictions.")
+                except Exception as e:
+                    logger.error(f"Failed to apply inverse transform: {e}", exc_info=True)
+                    # Decide whether to return scaled or None. Returning None is safer.
+                    return None
+
+            # 5. Interpolate
+            # Use the last actual price from the input data as the anchor point t=0
+            last_actual_price = historical_data_slice[self.target_col].iloc[-1]
+            interpolated_forecast = interpolate_forecast(
+                native_horizons=self.forecast_horizons,
+                native_predictions=predictions_original_scale,
+                target_horizon=optimization_horizon_hours,
+                last_known_actual=last_actual_price
+            )
+
+            if interpolated_forecast is None:
+                logger.error("Interpolation step failed.")
+                return None
+
+            logger.debug(f"SingleModelProvider: Successfully generated forecast.")
+            return interpolated_forecast
+
+        except Exception as e:
+            logger.error(f"Error during single model forecast generation: {e}", exc_info=True)
+            return None

optimizer/forecasting/utils.py

@@ -0,0 +1,67 @@
+from typing import List, Optional, Dict, Any
+
+import numpy as np
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+# --- Interpolation Helper ---
+def interpolate_forecast(
+    native_horizons: List[int],
+    native_predictions: np.ndarray,
+    target_horizon: int,
+    last_known_actual: Optional[float] = None # Optional: use last known price as t=0 for anchor
+) -> np.ndarray | None:
+    """
+    Linearly interpolates model predictions at native horizons to a full hourly sequence.
+
+    Args:
+        native_horizons: List of horizons the model predicts (e.g., [1, 6, 12, 24]). Must not be empty.
+        native_predictions: Numpy array of predictions corresponding to native_horizons. Must not be empty.
+        target_horizon: The desired length of the hourly forecast (e.g., 24).
+        last_known_actual: Optional last actual price before the forecast starts (at t=0). Used as anchor if 0 not in native_horizons.
+
+    Returns:
+        A numpy array of shape (target_horizon,) with interpolated values, or None on error.
+    """
+    if not native_horizons or native_predictions is None or native_predictions.size == 0:
+        logger.error("Cannot interpolate with empty native horizons or predictions.")
+        return None
+    if len(native_horizons) != len(native_predictions):
+        logger.error(f"Mismatched lengths: native_horizons ({len(native_horizons)}) vs native_predictions ({len(native_predictions)})")
+        return None
+
+    try:
+        # Ensure horizons are sorted
+        sorted_indices = np.argsort(native_horizons)
+        # Use float for potentially non-integer horizons if ever needed, ensure points > 0 usually
+        xp = np.array(native_horizons, dtype=float)[sorted_indices]
+        fp = native_predictions[sorted_indices]
+
+        # Target points for interpolation (hours 1 to target_horizon)
+        x_target = np.arange(1, target_horizon + 1, dtype=float)
+
+        # Add t=0 point if provided and 0 is not already a native horizon
+        # This anchors the start of the interpolation.
+        if last_known_actual is not None and xp[0] > 0:
+            xp = np.insert(xp, 0, 0.0)
+            fp = np.insert(fp, 0, last_known_actual)
+        elif xp[0] == 0 and last_known_actual is not None:
+             logger.debug("Native horizons include 0, using model's prediction for t=0 instead of last_known_actual.")
+        elif last_known_actual is None and xp[0] > 0:
+             logger.warning("No last_known_actual provided and native horizons start > 0. Interpolation might be less accurate at the beginning.")
+             # If the first native horizon is > 1, np.interp will extrapolate constantly backwards from the first point.
+
+
+        # Check if target range requires extrapolation beyond the model's capability
+        if target_horizon > xp[-1]:
+            logger.warning(f"Target horizon ({target_horizon}) extends beyond the maximum native forecast horizon ({xp[-1]}). Extrapolation will occur (constant value).")
+
+        interpolated_values = np.interp(x_target, xp, fp)
+        return interpolated_values
+
+    except Exception as e:
+        logger.error(f"Linear interpolation failed: {e}", exc_info=True)
+        return None

optimizer/optimization/battery.py

@@ -0,0 +1,74 @@
+import cvxpy as cp
+import numpy as np
+
+
+def solve_battery_optimization_hourly(
+    hourly_prices,      # Array of prices for each hour [0, 1, ..., n-1]
+    initial_B,        # Current state of charge (MWh)
+    max_capacity=1.0,   # MWh
+    max_rate=1.0        # MW (+ve discharge / -ve charge)
+):
+    """
+    Solves the battery scheduling optimization problem assuming hourly steps. We want to decide at the start of each hour t=0..n-1
+    how much power to buy/sell (P_net_t) and therefore the state of charge at the start of each next hour (B_t+1).
+
+    Args:
+        hourly_prices: Prices (€/MWh) for each hour t=0..n-1.
+        initial_B: The state of charge at the beginning (time t=0).
+        max_capacity: Maximum battery energy capacity (MWh).
+        max_rate: Maximum charge/discharge power rate (MW).
+
+    Returns:
+        Tuple: (status, optimal_profit, power_schedule, B_schedule)
+               Returns (status, None, None, None) if optimization fails.
+    """
+    n_hours = len(hourly_prices)
+
+    # --- CVXPY Variables ---
+    # Power flow for each hour t=0..n-1 (-discharge, +charge)
+    P = cp.Variable(n_hours, name="Power_Flow_MW")
+    # State of charge at the START of each hour t=0..n (B[t] is B at hour t)
+    B = cp.Variable(n_hours + 1, name="State_of_Charge_MWh")
+
+    # --- Objective Function ---
+    # Profit = sum(price[t] * Power[t])
+    prices = np.array(hourly_prices)
+    profit = prices @ P # Equivalent to cp.sum(cp.multiply(prices, P)) / prices.dot(P)
+    objective = cp.Maximize(profit)
+
+    # --- Constraints ---
+    constraints = []
+
+    # 1. Initial B
+    constraints.append(B[0] == initial_B)
+
+    # 2. B Dynamics: B[t+1] = B[t] - P[t] * 1 hour
+    constraints.append(B[1:] == B[:-1] + P)
+
+    # 3. Power Rate Limits: -max_rate <= P[t] <= max_rate
+    constraints.append(cp.abs(P) <= max_rate)
+
+    # 4. B Limits: 0 <= B[t] <= max_capacity (applies to B[0]...B[n])
+    constraints.append(B >= 0)
+    constraints.append(B <= max_capacity)
+
+    # --- Problem Definition and Solving ---
+    problem = cp.Problem(objective, constraints)
+    try:
+        # Alternative solvers are ECOS, MOSEK, and SCS
+        optimal_profit = problem.solve(solver=cp.CLARABEL, verbose=False)
+
+        if problem.status in [cp.OPTIMAL, cp.OPTIMAL_INACCURATE]:
+            return (
+                problem.status,
+                optimal_profit,
+                P.value,  # NumPy array of optimal power flows per hour
+                B.value   # NumPy array of optimal B at start of each hour
+            )
+        else:
+            print(f"Optimization failed. Solver status: {problem.status}")
+            return problem.status, None, None, None
+
+    except cp.error.SolverError as e:
+        print(f"Solver Error: {e}")
+        return "Solver Error", None, None, None

optimizer/optimzer_plan.md

@@ -0,0 +1,41 @@
+
+## Optimizer Definition for a constraint n-forecast Trading-Problem
+
+
+We want to optimize the performance of an energy trader given the forecast for n steps. 
+The battery:
+ - holds 1MWh
+ - charges/discharges at max. 1MW per hour (we can add/loose x*1MW, x \in R )
+Prices are stable for the given hour (t) and we sell and buy for the same price.
+
+
+### Considerations:
+ - Single variable, P (=x), for each hour t from 0 to n-1.
+
+ - If P > 0, it represents discharging (selling power) with a magnitude of P.
+ - If P < 0, it represents charging (buying power) with a magnitude of -P.
+ - If P = 0, it represents holding (doing nothing).
+
+ - if we have forecasts for t_n, t_n+m we might have to **interpolate** between n .. m
+ - or... we work with the gaps and dt as charge time .... no
+
+#### Variables:
+
+ - price_t = price per MWH at t (eq)
+ - B (t=0..n) = State of Battery in MWH
+ - P (t=0..n) = Charge/Discharge factor given the possible base rate of 1MW/h
+ - max_p = 1 (charge/discharge limits) & and battery capacity limits (both=1)
+ - SoB_initial = 0
+ - h = horizon \in N^+
+
+
+### Objective
+ - We **Maximize**: Sum_{t=0}^{n-1} (price_t * P)
+
+### Constraints
+ - Fixed starting state: SoB_0 = SoB_initial
+ - Charge/Discharge Limit: (-max_p <= P <= max_p) for all t = 0, ..., n-1
+ - Storage Limit: (0 <= B+(1*P) <= max_p) for all t = 0, ..., n-1
+ - Future B State: SoB_{t+1} = (B + P) for t = 0 to n-1
+
+

optimizer/utils/model_io.py

@@ -0,0 +1,297 @@
+import logging
+import yaml
+import json
+from pathlib import Path
+from typing import Dict, Any, Optional, List
+
+import torch
+from sklearn.base import BaseEstimator, TransformerMixin # For scaler type hint
+
+# Import necessary components from forecasting_model
+from forecasting_model.utils.forecast_config_model import MainConfig, FeatureConfig
+from forecasting_model.train.model import LSTMForecastLightningModule
+
+logger = logging.getLogger(__name__)
+
+def load_single_model_artifact(
+    model_path: Path,
+    config_path: Path,
+    input_size_path: Path,
+    target_scaler_path: Optional[Path] = None
+) -> Optional[Dict[str, Any]]:
+    """
+    Loads artifacts for a single trained model checkpoint.
+
+    Args:
+        model_path: Path to the model checkpoint file (.ckpt).
+        config_path: Path to the corresponding main YAML config file.
+        input_size_path: Path to the input_size.pt file.
+        target_scaler_path: Optional path to the target_scaler.pt file.
+
+    Returns:
+        A dictionary containing loaded artifacts ('model_instance', 'feature_config',
+        'target_scaler', 'main_forecasting_config'), or None if loading fails.
+    """
+    logger.info(f"Loading single model artifact from directory: {model_path.parent}")
+    loaded_artifacts = {}
+
+    try:
+        # 1. Load Config
+        if not config_path.is_file():
+            logger.error(f"Config file not found at {config_path}")
+            return None
+        with open(config_path, 'r') as f:
+            config_data = yaml.safe_load(f)
+        main_config = MainConfig(**config_data)
+        loaded_artifacts['main_forecasting_config'] = main_config
+        loaded_artifacts['feature_config'] = main_config.features
+        logger.debug(f"Loaded config from {config_path}")
+
+        # 2. Load Input Size
+        if not input_size_path.is_file():
+            logger.error(f"Input size file not found at {input_size_path}")
+            return None
+        input_size = torch.load(input_size_path)
+        if not isinstance(input_size, int) or input_size <= 0:
+            logger.error(f"Invalid input size loaded from {input_size_path}: {input_size}")
+            return None
+        logger.debug(f"Loaded input size ({input_size}) from {input_size_path}")
+
+        # 3. Load Target Scaler (Optional)
+        target_scaler = None
+        if target_scaler_path:
+            if not target_scaler_path.is_file():
+                logger.warning(f"Target scaler file not found at {target_scaler_path}. Proceeding without scaler.")
+            else:
+                try:
+                    target_scaler = torch.load(target_scaler_path)
+                    # Basic check if it looks like a scaler
+                    if not isinstance(target_scaler, (BaseEstimator, TransformerMixin)):
+                         logger.warning(f"Loaded object from {target_scaler_path} might not be a valid scaler ({type(target_scaler)}).")
+                         # Decide if this should be a hard failure or just a warning
+                    else:
+                         logger.debug(f"Loaded target scaler from {target_scaler_path}")
+                except Exception as e:
+                    logger.error(f"Error loading target scaler from {target_scaler_path}: {e}", exc_info=True)
+                    # Decide if this should be a hard failure
+                    return None # Fail hard if scaler loading fails
+        loaded_artifacts['target_scaler'] = target_scaler
+
+        # 4. Initialize Model Architecture
+        # Ensure model config forecast horizon matches feature config (should be guaranteed by MainConfig validation)
+        if set(main_config.model.forecast_horizon) != set(main_config.features.forecast_horizon):
+             logger.warning(f"Mismatch between model ({main_config.model.forecast_horizon}) and feature ({main_config.features.forecast_horizon}) forecast horizons in config {config_path}. Using feature config.")
+             # This might indicate an issue with the saved config, but we proceed using the feature config horizon
+             # main_config.model.forecast_horizon = main_config.features.forecast_horizon # Correct it for model init? Risky.
+
+        model_instance = LSTMForecastLightningModule(
+             model_config=main_config.model,
+             train_config=main_config.training, # Pass train config if needed
+             input_size=input_size,
+             target_scaler=target_scaler # Pass scaler to model if it uses it internally during inference
+        )
+        logger.debug("Initialized model architecture.")
+
+        # 5. Load Model State Dictionary
+        if not model_path.is_file():
+            logger.error(f"Model checkpoint file not found at {model_path}")
+            return None
+        # Load onto CPU first to avoid GPU memory issues if the loading machine is different
+        state_dict = torch.load(model_path, map_location=torch.device('cpu'))
+        # Adjust state dict keys if saved with 'model.' prefix from Lightning wrapper common during saving ckpt
+        if any(key.startswith('model.') for key in state_dict.get('state_dict', state_dict).keys()):
+            state_dict = {k.partition('model.')[2]: v for k, v in state_dict.get('state_dict', state_dict).items()}
+            logger.debug("Adjusted state dict keys (removed 'model.' prefix).")
+
+        # Load the state dict
+        # Use strict=False initially if unsure about exact key matching, but strict=True is safer
+        try:
+             load_result = model_instance.load_state_dict(state_dict, strict=True)
+             logger.debug(f"Model state loaded. Result: {load_result}")
+        except RuntimeError as e:
+             logger.error(f"Error loading state dict into model (strict=True): {e}. Trying strict=False.")
+             try:
+                 load_result = model_instance.load_state_dict(state_dict, strict=False)
+                 logger.warning(f"Model state loaded with strict=False. Result: {load_result}. Check for missing/unexpected keys.")
+             except Exception as e_false:
+                 logger.error(f"Failed to load state dict even with strict=False: {e_false}", exc_info=True)
+                 return None
+
+
+        model_instance.eval() # Set model to evaluation mode
+        loaded_artifacts['model_instance'] = model_instance
+        logger.info(f"Successfully loaded single model artifact: {model_path.name}")
+
+        return loaded_artifacts
+
+    except FileNotFoundError:
+         logger.error(f"A required file was not found during artifact loading for {model_path.parent}.", exc_info=True)
+         return None
+    except yaml.YAMLError as e:
+         logger.error(f"Error parsing YAML config file {config_path}: {e}", exc_info=True)
+         return None
+    except Exception as e:
+        logger.error(f"Failed to load single model artifact from {model_path.parent}: {e}", exc_info=True)
+        return None
+
+
+def load_ensemble_artifact(
+    ensemble_definition_path: Path,
+    hpo_base_output_dir: Path # Base directory where HPO study results (including ensemble JSON) are saved
+) -> Optional[Dict[str, Any]]:
+    """
+    Loads artifacts for an ensemble based on its definition JSON file.
+
+    Args:
+        ensemble_definition_path: Path to the _best_ensemble.json file.
+        hpo_base_output_dir: The base directory where the HPO study ran and
+                             where relative paths within the JSON are anchored.
+
+    Returns:
+        A dictionary containing 'ensemble_method', 'fold_artifacts' (a list
+        of dictionaries, each like the output of load_single_model_artifact),
+        'ensemble_feature_config', and 'ensemble_target_col', or None if loading fails.
+    """
+    logger.info(f"Loading ensemble artifact definition from: {ensemble_definition_path}")
+
+    try:
+        if not ensemble_definition_path.is_file():
+            logger.error(f"Ensemble definition file not found at: {ensemble_definition_path}")
+            return None
+        with open(ensemble_definition_path, 'r') as f:
+            ensemble_definition = json.load(f)
+    except json.JSONDecodeError as e:
+        logger.error(f"Error decoding ensemble definition JSON file: {e}", exc_info=True)
+        return None
+    except Exception as e:
+        logger.error(f"Error loading ensemble definition: {e}", exc_info=True)
+        return None
+
+    # Extract information from the definition
+    ensemble_method = ensemble_definition.get("ensemble_method")
+    fold_models_definitions = ensemble_definition.get("fold_models")
+    # Base directory for artifacts *relative to* hpo_base_output_dir
+    relative_artifacts_base_dir = ensemble_definition.get("ensemble_artifacts_base_dir")
+
+    if not ensemble_method or not fold_models_definitions:
+         logger.error("Ensemble definition file is missing 'ensemble_method' or 'fold_models' list.")
+         return None
+    if not relative_artifacts_base_dir:
+         logger.error("Ensemble definition file is missing 'ensemble_artifacts_base_dir'. Cannot locate fold artifacts.")
+         return None
+
+    # --- Determine Absolute Path to Fold Artifacts ---
+    # The paths inside fold_models are relative to ensemble_artifacts_base_dir,
+    # which itself is relative to hpo_base_output_dir.
+    absolute_artifacts_base_dir = hpo_base_output_dir / Path(relative_artifacts_base_dir)
+    logger.debug(f"Absolute base directory for fold artifacts: {absolute_artifacts_base_dir}")
+    if not absolute_artifacts_base_dir.is_dir():
+        logger.error(f"Calculated absolute artifact base directory does not exist or is not a directory: {absolute_artifacts_base_dir}")
+        return None
+
+
+    loaded_fold_artifacts: List[Dict[str, Any]] = []
+    common_feature_config: Optional[FeatureConfig] = None
+    common_target_col: Optional[str] = None
+
+    logger.info(f"Loading artifacts for {len(fold_models_definitions)} folds defined in the ensemble...")
+
+    # --- Load Artifacts for Each Fold ---
+    for i, fold_def in enumerate(fold_models_definitions):
+        fold_id = fold_def.get("fold_id", i + 1)
+        logger.debug(f"--- Loading Fold {fold_id} ---")
+
+        model_path_rel = fold_def.get("model_path")
+        scaler_path_rel = fold_def.get("target_scaler_path")
+        input_size_path_rel = fold_def.get("input_size_path")
+        config_path_rel = fold_def.get("config_path")
+
+        if not model_path_rel or not input_size_path_rel or not config_path_rel:
+             logger.error(f"Fold {fold_id}: Definition is missing required path(s) (model, input_size, or config). Skipping fold.")
+             continue
+
+        # Construct absolute paths for this fold's artifacts
+        try:
+            abs_model_path = (absolute_artifacts_base_dir / Path(model_path_rel)).resolve()
+            abs_input_size_path = (absolute_artifacts_base_dir / Path(input_size_path_rel)).resolve()
+            abs_config_path = (absolute_artifacts_base_dir / Path(config_path_rel)).resolve()
+            abs_scaler_path = (absolute_artifacts_base_dir / Path(scaler_path_rel)).resolve() if scaler_path_rel else None
+
+            logger.debug(f"Fold {fold_id} - Model Path: {abs_model_path}")
+            logger.debug(f"Fold {fold_id} - Config Path: {abs_config_path}")
+            logger.debug(f"Fold {fold_id} - Input Size Path: {abs_input_size_path}")
+            logger.debug(f"Fold {fold_id} - Scaler Path: {abs_scaler_path}")
+
+            # Load the artifacts for this single fold using the other function
+            single_fold_loaded_artifacts = load_single_model_artifact(
+                model_path=abs_model_path,
+                config_path=abs_config_path,
+                input_size_path=abs_input_size_path,
+                target_scaler_path=abs_scaler_path
+            )
+
+            if single_fold_loaded_artifacts:
+                # Add fold_id for reference
+                single_fold_loaded_artifacts['fold_id'] = fold_id
+                loaded_fold_artifacts.append(single_fold_loaded_artifacts)
+                logger.info(f"Successfully loaded artifacts for fold {fold_id}.")
+
+                # --- Consistency Check (Optional but Recommended) ---
+                # Store the feature config and target col from the first successful fold
+                # Then compare subsequent folds against these
+                current_feature_config = single_fold_loaded_artifacts['feature_config']
+                current_target_col = single_fold_loaded_artifacts['main_forecasting_config'].data.target_col
+
+                if common_feature_config is None:
+                    common_feature_config = current_feature_config
+                    common_target_col = current_target_col
+                    logger.debug(f"Set common feature config and target column based on fold {fold_id}.")
+                else:
+                    # Compare crucial feature engineering aspects
+                    if common_feature_config.sequence_length != current_feature_config.sequence_length or \
+                       set(common_feature_config.forecast_horizon) != set(current_feature_config.forecast_horizon) or \
+                       common_feature_config.scaling_method != current_feature_config.scaling_method: # Add more checks if needed
+                        logger.error(f"Fold {fold_id}: Feature configuration mismatch with previous folds. Cannot proceed with this ensemble definition.")
+                        # You might want to compare more fields like lags, rolling_windows etc.
+                        return None # Fail hard if configs are inconsistent
+                    if common_target_col != current_target_col:
+                         logger.error(f"Fold {fold_id}: Target column '{current_target_col}' differs from previous folds ('{common_target_col}'). Cannot proceed.")
+                         return None # Fail hard
+
+            else:
+                 logger.error(f"Failed to load artifacts for fold {fold_id}. Skipping fold.")
+                 # Decide if ensemble loading should fail if *any* fold fails
+                 # For now, we continue and will check if enough folds loaded later
+
+        except TypeError as e:
+             # Catch potential errors if paths are None or invalid types
+             logger.error(f"Fold {fold_id}: Error constructing artifact paths - check definition file content: {e}", exc_info=True)
+             continue
+        except Exception as e:
+            logger.error(f"Fold {fold_id}: Unexpected error during loading: {e}", exc_info=True)
+            continue # Skip this fold
+
+    # --- Final Checks and Return ---
+    if not loaded_fold_artifacts:
+        logger.error("Failed to load artifacts for *any* fold in the ensemble.")
+        return None
+
+    # Add a check if a minimum number of folds is required (e.g., > 1)
+    if len(loaded_fold_artifacts) < 1: # Or maybe check against len(fold_models_definitions)?
+         logger.error(f"Only successfully loaded {len(loaded_fold_artifacts)} folds, which might be insufficient for the ensemble.")
+         # Decide if this is an error or just a warning
+         return None
+
+    if common_feature_config is None or common_target_col is None:
+         # This should not happen if loaded_fold_artifacts is not empty, but check anyway
+         logger.error("Internal error: Could not determine common feature config or target column for the ensemble.")
+         return None
+
+    logger.info(f"Successfully loaded artifacts for {len(loaded_fold_artifacts)} ensemble folds.")
+
+    return {
+        'ensemble_method': ensemble_method,
+        'fold_artifacts': loaded_fold_artifacts, # List of dicts
+        'ensemble_feature_config': common_feature_config, # The common config
+        'ensemble_target_col': common_target_col # The common target column name
+    }

optimizer/utils/optim_config.py

@@ -0,0 +1,18 @@
+from pydantic import BaseModel, Field
+from typing import List, Optional, Literal
+
+class ModelEvalConfig(BaseModel):
+    """Configuration for evaluating a single forecasting model or an ensemble."""
+    name: str = Field(..., description="Name of the forecasting model or ensemble.")
+    type: Literal['model', 'ensemble'] = Field(..., description="Type of evaluation artifact: 'model' for a single checkpoint, 'ensemble' for an ensemble definition JSON.")
+    model_path: str = Field(..., description="Path to the saved PyTorch model file (.ckpt for type='model') or the ensemble definition JSON file (.json for type='ensemble').")
+    model_config_path: str = Field(..., description="Path to the forecasting config (YAML) used for this model training (or for the best trial in an ensemble).")
+    target_scaler_path: Optional[str] = Field(None, description="Path to the target scaler file for the single model (or will be loaded per fold for ensemble).")
+
+class OptimizationRunConfig(BaseModel):
+    """Main configuration for running battery optimization with multiple models/ensembles."""
+    initial_b: float = Field(..., description="Initial state of charge of the battery (MWh).")
+    max_capacity: float = Field(..., description="Maximum energy capacity of the battery (MWh).")
+    max_rate: float = Field(..., description="Maximum charge/discharge power rate of the battery (MW).")
+    optimization_horizon_hours: int = Field(24, gt=0, description="The length of the time window (in hours) for optimization.")
+    models: List[ModelEvalConfig] = Field(..., description="List of forecasting models or ensembles to evaluate.")