"""Case 002: singular random-effects covariance, NumPy + SciPy.
Download case-002-shared-data.csv beside this script.
Educational REML fit for this balanced Gaussian intercept/slope example.
Uses a covariance Cholesky parameterization permitting zero dimensions.
Not a general replacement for lme4; its eigenvalue diagnostic is distinct
from lme4's tolerance-based isSingular/rePCA implementation.
"""
import csv
from pathlib import Path
import numpy as np
import scipy
from scipy.optimize import minimize

with Path(__file__).with_name('case-002-shared-data.csv').open() as f:
    rows = list(csv.DictReader(f))
groups = sorted({r['group'] for r in rows}, key=int)
ys = np.array([[float(r['y']) for r in rows if r['group']==g] for g in groups])
x = np.array([float(r['x']) for r in rows if r['group']==groups[0]])
X = np.column_stack([np.ones(len(x)), x])
J, m = ys.shape

def components(p, slope=True):
    if slope:
        a, b, c, log_sd = p
        L = np.array([[a, 0], [b, c]])
        G = L @ L.T
    else:
        a, log_sd = p
        G = np.array([[a*a, 0], [0, 0]])
    V = X @ G @ X.T + np.exp(2*log_sd)*np.eye(m)
    Vi = np.linalg.inv(V)
    information = J * X.T @ Vi @ X
    beta = np.linalg.solve(information, X.T @ Vi @ ys.sum(axis=0))
    residual = ys - X @ beta
    objective = .5*(J*np.linalg.slogdet(V)[1] +
                     np.linalg.slogdet(information)[1] +
                     np.einsum('ji,ik,jk->', residual, Vi, residual))
    return objective, G, beta

def fit(slope):
    starts = [[1, .02, .1, 0], [1, 0, 0, 0]] if slope else [[1, 0]]
    bounds = [(0,None),(None,None),(0,None),(-5,5)] if slope else [(0,None),(-5,5)]
    results = [minimize(lambda p: components(p,slope)[0], start,
               method='L-BFGS-B', jac='3-point', bounds=bounds,
               options={'ftol':1e-13,'gtol':1e-6,'maxiter':2000,'maxls':50}) for start in starts]
    return min(results,key=lambda r:r.fun)

full = fit(True)
supported = fit(False)
_, G, beta = components(full.x)
eigen = np.linalg.eigvalsh(G)
correlation = G[0,1]/np.sqrt(G[0,0]*G[1,1])
_, _, beta_supported = components(supported.x,False)
print('NumPy',np.__version__,'SciPy',scipy.__version__)
print('Optimizer success:',full.success,full.message)
print('Random-effects covariance:',G)
print('Eigenvalues:',eigen)
print('Intercept-slope correlation:',correlation)
print('Full and intercept-only fixed slopes:',beta[1],beta_supported[1])
print('A completed optimizer can coexist with a rank-deficient covariance.')
print('Simplification is justified here by the known generating process only.')
assert full.success and supported.success
assert eigen.min()/eigen.max() < 1e-6
assert abs(correlation) > .999
assert abs(beta[1]-.578134) < 1e-5
assert abs(beta[1]-beta_supported[1]) < 1e-7
print('All checks passed.')
