- second commit

src/optimize.rs

@@ -0,0 +1,167 @@
+use std::collections::VecDeque;
+use std::fmt::Debug;
+use algos::{ObjectiveFunction, OptimizationConfig, OptimizationResult};
+use num::Float;
+
+pub fn lbfgs_minimize<T, F>(
+    f: &F,
+    initial_point: &[T],
+    config: &OptimizationConfig<T>,
+) -> OptimizationResult<T>
+where
+    T: Float + Debug,
+    F: ObjectiveFunction<T>,
+{
+    const M: usize = 10; // Number of corrections to store
+    let n = initial_point.len();
+    let mut current_point = initial_point.to_vec();
+    let mut iterations = 0;
+    let mut converged = false;
+
+    // Storage for the last M corrections
+    let mut s_list: VecDeque<Vec<T>> = VecDeque::with_capacity(M);
+    let mut y_list: VecDeque<Vec<T>> = VecDeque::with_capacity(M);
+    let mut rho_list: VecDeque<T> = VecDeque::with_capacity(M);
+
+    // Get initial gradient
+    let mut gradient = match f.gradient(&current_point) {
+        Some(g) => g,
+        None => {
+            return OptimizationResult {
+                optimal_point: current_point.clone(),
+                optimal_value: f.evaluate(&current_point),
+                iterations: 0,
+                converged: false,
+            };
+        }
+    };
+
+    while iterations < config.max_iterations {
+        // Check for convergence
+        let gradient_norm = gradient
+            .iter()
+            .fold(T::zero(), |acc, &x| acc + x * x)
+            .sqrt();
+        if gradient_norm < config.tolerance {
+            converged = true;
+            break;
+        }
+
+        // Compute search direction using L-BFGS two-loop recursion
+        let mut q = gradient.clone();
+        let mut alpha_list = Vec::with_capacity(s_list.len());
+
+        // First loop
+        for i in (0..s_list.len()).rev() {
+            let alpha = rho_list[i]
+                * s_list[i]
+                .iter()
+                .zip(q.iter())
+                .fold(T::zero(), |acc, (&s, &q)| acc + s * q);
+            alpha_list.push(alpha);
+            for (q_j, y_j) in q.iter_mut().zip(y_list[i].iter()) {
+                *q_j = *q_j - alpha * *y_j;
+            }
+        }
+
+        // Scale the initial Hessian approximation
+        let mut r = if !s_list.is_empty() {
+            let i = s_list.len() - 1;
+            let yy = y_list[i].iter().fold(T::zero(), |acc, &y| acc + y * y);
+            let ys = y_list[i]
+                .iter()
+                .zip(s_list[i].iter())
+                .fold(T::zero(), |acc, (&y, &s)| acc + y * s);
+            q.iter_mut().for_each(|r_j| *r_j = *r_j * (ys / yy));
+            q
+        } else {
+            q.iter_mut()
+                .for_each(|r_j| *r_j = *r_j * config.learning_rate);
+            q
+        };
+
+        // Second loop
+        for i in 0..s_list.len() {
+            let beta = rho_list[i]
+                * y_list[i]
+                .iter()
+                .zip(r.iter())
+                .fold(T::zero(), |acc, (&y, &r)| acc + y * r);
+            let alpha = alpha_list[s_list.len() - 1 - i];
+            for (r_j, s_j) in r.iter_mut().zip(s_list[i].iter()) {
+                *r_j = *r_j + (alpha - beta) * *s_j;
+            }
+        }
+
+        // r now contains the search direction
+        let direction: Vec<T> = r.iter().map(|&x| -x).collect();
+
+        // Line search to find step size
+        let mut alpha = T::one();
+        let mut new_point = vec![T::zero(); n];
+        let current_value = f.evaluate(&current_point);
+
+        // Simple backtracking line search
+        for _ in 0..20 {
+            for i in 0..n {
+                new_point[i] = current_point[i] + alpha * direction[i];
+            }
+            let new_value = f.evaluate(&new_point);
+            if new_value < current_value {
+                break;
+            }
+            alpha = alpha * T::from(0.5).unwrap();
+        }
+
+
+        // Get new gradient
+        let new_gradient = match f.gradient(&new_point) {
+            Some(g) => g,
+            None => break,
+        };
+
+        // Update the correction vectors
+        let s = new_point
+            .iter()
+            .zip(current_point.iter())
+            .map(|(&x_new, &x_old)| x_new - x_old)
+            .collect::<Vec<T>>();
+        let y = new_gradient
+            .iter()
+            .zip(gradient.iter())
+            .map(|(&g_new, &g_old)| g_new - g_old)
+            .collect::<Vec<T>>();
+
+        let ys = y
+            .iter()
+            .zip(s.iter())
+            .fold(T::zero(), |acc, (&y_i, &s_i)| acc + y_i * s_i);
+
+        if ys == T::zero() {
+            break;
+        }
+
+        let rho = T::one() / ys;
+
+        if s_list.len() == M {
+            s_list.pop_front();
+            y_list.pop_front();
+            rho_list.pop_front();
+        }
+        s_list.push_back(s);
+        y_list.push_back(y);
+        rho_list.push_back(rho);
+        
+        // Update for next iteration
+        current_point = new_point;
+        gradient = new_gradient;
+        iterations += 1;
+    }
+
+    OptimizationResult {
+        optimal_point: current_point.clone(),
+        optimal_value: f.evaluate(&current_point),
+        iterations,
+        converged,
+    }
+}