Equivalent LoRA updates can break factor averaging

"""Reproducible algebra checks, Python 3 standard library only. No model calls. Run: python3 numerical_checks.py Outputs: results.json and lora-factor-error.svg next to this script. """ from pathlib import Path import json import math import random import statistics OUT = Path(__file__).resolve().parent def mm(a, b): return [[sum(x*y for x, y in zip(row, col)) for col in zip(*b)] for row in a] def scale(a, c): return [[c*x for x in row] for row in a] def mean(a, b): return [[(x+y)/2 for x, y in zip(ra, rb)] for ra, rb in zip(a, b)] def error(a, target): return math.sqrt(sum((x-y)**2 for ra, rb in zip(a, target) for x, y in zip(ra, rb)) / sum(x*x for row in target for x in row)) rows = [] for c in [1, 2, 4, 8, 16, -1]: naive, dense = [], [] for seed in range(100): rng = random.Random(seed) b = [[rng.gauss(0, 1) for _ in range(4)] for _ in range(16)] a = [[rng.gauss(0, 1) for _ in range(16)] for _ in range(4)] target = mm(b, a) b2, a2 = scale(b, c), scale(a, 1/c) dense.append(error(mean(target, mm(b2, a2)), target)) naive.append(error(mm(mean(b, b2), mean(a, a2)), target)) expected = abs((2+c+1/c)/4-1) assert max(abs(v-expected) for v in naive) < 1e-12 assert max(dense) < 1e-12 rows.append(dict(scale=c, seeds=100, factor_relative_error_median=statistics.median(naive), delta_relative_error_max=max(dense), analytic_relative_error=expected)) # Independent forward-sum vs backward-recursion GAE comparison. max_gae_error = 0.0 max_projection_error = 0.0 for seed in range(100): rng = random.Random(seed) for length in [1, 2, 8, 32]: reward = [0.0]*(length-1) + [rng.uniform(-2, 2)] values = [rng.uniform(-1, 1) for _ in range(length)] + [0.0] gamma, lam = 0.99, 0.95 deltas = [reward[t]+gamma*values[t+1]-values[t] for t in range(length)] direct = [sum((gamma*lam)**k * deltas[t+k] for k in range(length-t)) for t in range(length)] backward = [0.0]*length carry = 0.0 for t in reversed(range(length)): carry = deltas[t]+gamma*lam*carry backward[t] = carry max_gae_error = max(max_gae_error, max(abs(a-b) for a, b in zip(direct, backward))) phi = [rng.uniform(-1, 1) for _ in range(length)] z = rng.uniform(-2, 2) projected = [v+(z-sum(phi))/length for v in phi] max_projection_error = max(max_projection_error, abs(sum(projected)-z)) assert max_gae_error < 1e-12 assert max_projection_error < 1e-12 result = dict(scope='Synthetic linear algebra and estimator identity checks; no trained LLM or task benchmark', seeds=list(range(100)), matrix_shape=[16,16], lora_rank=4, precision='Python float (64-bit)', lora=rows, gae_cases=400, gae_max_absolute_error=max_gae_error, projection_cases=400, projection_max_absolute_error=max_projection_error) (OUT/'results.json').write_text(json.dumps(result, indent=2)+'\n') # A deterministic SVG; vertical error scale starts at zero. svg=['']) (OUT/'lora-factor-error.svg').write_text('\n'.join(svg)+'\n') print(json.dumps(result, indent=2))