\documentclass[11pt,a4paper]{article}

\usepackage[a4paper,margin=0.82in]{geometry}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,amsfonts,amsthm,mathtools,bm}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{caption}
\usepackage{subcaption}
\usepackage{float}
\usepackage{placeins}
\usepackage{xcolor}
\usepackage[colorlinks=true,linkcolor=blue,citecolor=blue,urlcolor=blue]{hyperref}

\graphicspath{{figures/}}

\newcommand{\E}{\mathbb E}
\newcommand{\Pp}{\mathbb P}
\newcommand{\R}{\mathbb R}
\newcommand{\N}{\mathcal N}
\newcommand{\Hh}{\mathcal H}
\newcommand{\Kk}{\mathcal K}
\newcommand{\Dd}{\mathcal D}
\newcommand{\Ll}{\mathcal L}
\newcommand{\Rr}{\mathcal R}
\newcommand{\Ee}{\mathcal E}
\newcommand{\Ff}{\mathcal F}
\newcommand{\Id}{I}
\newcommand{\eps}{\varepsilon}
\newcommand{\tr}{\operatorname{Tr}}
\newcommand{\rank}{\operatorname{rank}}
\newcommand{\diag}{\operatorname{diag}}
\newcommand{\op}{\operatorname{op}}
\newcommand{\argmin}{\operatorname*{arg\,min}}
\newcommand{\Span}{\operatorname{span}}
\newcommand{\T}{\top}

\newtheorem{theorem}{Theorem}
\newtheorem{proposition}{Proposition}
\newtheorem{lemma}{Lemma}
\newtheorem{definition}{Definition}
\newtheorem{remark}{Remark}

\title{\vspace{-2.0em}Transformers for In-Context PDE Solving and Inverse Problems\\
\large A self-contained finite-dimensional theory of encoder recovery, decoder preconditioning, generalization certificates, and replica order parameters}
\author{MMNN transformer/PDE notes}
\date{July 6, 2026}

\begin{document}
\maketitle

\begin{abstract}
We rewrite the PDE and inverse-problem in-context-learning task as finite
least squares.  One task is one latent vector \(z\); one prompt is many pairs
\((u_i,f_i)\) produced by that same \(z\).  The encoder estimates \(z\) from
weak equations \(G_mz\simeq b_m\).  The decoder solves the query system by a
learned preconditioned Richardson iteration.  This gives exact optimization
identities for encoder-only, decoder-only, and stop-gradient encoder--decoder
training.  Generalization is certified either by exact Gaussian/Wishart risk
formulas in the low-rank model or by clipped empirical Bernstein bounds on
held-out tasks.  The replica framework of low-rank linear-attention ICL applies
exactly to the linear-attention finite least-squares reduction; for nonlinear
or learnable-kernel attention it gives the correct order-parameter target, but
the global scalar closure is not yet proved.  The required order parameters are
not only spectra: they include spectral overlaps between the learned
preconditioner, the task Hessian, and the source covariance.
\end{abstract}

\tableofcontents

\section{One paragraph reading guide}

The problem has two clocks.  Outer time \(s\) is pretraining: the attention
matrices, encoder dictionary, or preconditioner change.  Inner time \(\ell\) is
in-context solving at frozen parameters: one decoder layer is one solver step.
The exact identities are:
\[
G_s-G_\star=(G_0-G_\star)(I-\eta\Sigma_z)^s,
\qquad
e_{\ell+1}=(I-P_sH)e_\ell,
\qquad
e_L=(I-P_sH)^Le_0.
\]
The statistical question is how many prompt equations, training tasks, and
latent task directions are needed for these identities to be useful on new
tasks.  The replica question is which macroscopic overlaps close the typical
test risk in the high-dimensional limit.

\section{From a PDE prompt to finite least squares}

\subsection{Task, prompt, and query}

Let \(z\in\R^r\) be the hidden task coordinate,
\[
z\sim\N(0,\Sigma_z).
\]
Let \(f\) be a Gaussian process in a Hilbert space \(\Hh\),
\[
f\sim\operatorname{GP}(0,k_f),
\]
and let the elliptic operator be affine in \(z\):
\[
A(z)=A_0+\sum_{k=1}^r z_kA_k .
\]
For one task \(z\), the prompt contains \(m\) source-solution pairs
\[
\Pi_z=\{(u_i,f_i)\}_{i=1}^m,
\qquad
A(z)u_i=f_i,
\qquad
f_i\stackrel{\rm iid}{\sim}\operatorname{GP}(0,k_f).
\]
The query is a fresh \(f_\star\) from the same source law, and the desired
answer is
\[
u_\star=A(z)^{-1}f_\star.
\]
Thus:
\[
\boxed{\text{one task}=\text{one latent }z=\text{one prompt with many pairs }(u_i,f_i).}
\]

\subsection{Weak equations for the encoder}

Choose test functions \(v_1,\dots,v_q\).  Since
\[
f_i-A_0u_i=\sum_{k=1}^r z_kA_ku_i,
\]
testing against \(v_a\) gives
\[
\langle v_a,f_i-A_0u_i\rangle
=\sum_{k=1}^r z_k\langle v_a,A_ku_i\rangle .
\]
Stack rows indexed by \((i,a)\):
\[
G_{(i,a),k}=\langle v_a,A_ku_i\rangle,
\qquad
b_{(i,a)}=\langle v_a,f_i-A_0u_i\rangle .
\]
Then the prompt is the finite inverse problem
\[
G_mz=b_m,\qquad G_m\in\R^{mq\times r}.
\]
With noise or model error,
\[
b_m=G_mz+\xi,\qquad \E[\xi\xi^\T]=\sigma^2I.
\]
The ridge encoder is
\[
\widehat z_\lambda=(G_m^\T G_m+\lambda I)^{-1}G_m^\T b_m.
\]

\subsection{Finite-rank GP/KRR recast}

If the GP has a finite feature representation
\[
f(x)=\phi(x)^\T z,\qquad z\sim\N(0,\Lambda),
\]
then observations \(y=\Phi z+\eps\), \(\eps\sim\N(0,\sigma^2I)\), give
\[
A_{\rm post}=\Lambda^{-1}+\sigma^{-2}\Phi^\T\Phi,
\qquad
d_{\rm post}=\sigma^{-2}\Phi^\T y,
\qquad
z_{\rm post}=A_{\rm post}^{-1}d_{\rm post}.
\]
The dual KRR predictor is exactly the same:
\[
\phi(x)^\T z_{\rm post}
=k_x^\T(K+\sigma^2I)^{-1}y,
\qquad
k(x,x')=\phi(x)^\T\Lambda\phi(x').
\]
This is the finite-dimensional bridge used by the GP/KRR experiments.

\begin{figure}[H]
\centering
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{gp_posterior_framework_comparison.png}
\caption{Finite-rank GP/KRR posterior solvers.}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{pde_encdec_framework_comparison.png}
\caption{PDE encoder--decoder recast.}
\end{subfigure}
\caption{The GP and PDE tasks reduce to latent least-squares systems.  The
reported dual--primal GP gap is \(1.443\times10^{-14}\); in the PDE K=4 recast
the encoder has \(z\)-MSE \(6.136\times10^{-8}\).}
\end{figure}

\section{Task diversity and task difficulty}

The task diversity is the effective dimension of \(\Sigma_z\):
\[
d_{\rm task}
=\frac{\tr(\Sigma_z)^2}{\tr(\Sigma_z^2)}.
\]
If \(\Sigma_z\) is rank \(r\) isotropic, then \(d_{\rm task}=r\).  If only a few
latent directions have large variance, then pretraining sees a smaller task
family even if the ambient parameter vector is long.

The prompt visibility matrix is
\[
F_m=\frac1mG_m^\T G_m.
\]
The easy regime is
\[
\lambda_{\min}(F_m)\gg 0,
\qquad
\kappa(F_m+\lambda I)\ \text{moderate}.
\]
The hard regime is when some latent directions are invisible in the prompt:
\[
\lambda_{\min}(F_m)\simeq 0.
\]
The decoder difficulty for a known task is
\[
\kappa(H(z))=\frac{\lambda_{\max}(H(z))}{\lambda_{\min}(H(z))},
\]
where \(H(z)\) is the query Galerkin or latent least-squares Hessian.

If the whitened prompt rows are iid Gaussian and \(n=mq\), then
\[
F_m\sim \frac1nX^\T X,\qquad X_{ij}\sim\N(0,1).
\]
For \(r,n\to\infty\) with \(\gamma=r/n\), the empirical spectrum converges to
Marchenko--Pastur:
\[
d\mu_{\rm MP,\gamma}(\lambda)
=\frac{\sqrt{(\lambda_+-\lambda)(\lambda-\lambda_-)}}{2\pi\gamma\lambda}
\,\mathbf 1_{\lambda\in[\lambda_-,\lambda_+]}\,d\lambda,
\qquad
\lambda_\pm=(1\pm\sqrt\gamma)^2.
\]
Thus the prompt-size scaling is controlled by \(\gamma=r/(mq)\).

\begin{figure}[H]
\centering
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_prompt_mp_spectrum_fit.png}
\caption{Prompt spectrum and MP window.}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_replica_prompt_aspect_scaling.png}
\caption{Prompt aspect-ratio scaling.}
\end{subfigure}
\caption{Prompt-size laws.  Increasing the number of prompt pairs \(m\) or weak
tests \(q\) reduces \(\gamma=r/(mq)\), opens the smallest eigenvalue, and
improves encoder recovery.}
\end{figure}

\section{Encoder-only theory}

\subsection{Population model}

The encoder-only low-rank recovery model is
\[
b=G_\star z+\xi,\qquad z\sim\N(0,\Sigma_z),\qquad \xi\sim\N(0,\sigma^2I).
\]
The learned matrix \(G\) minimizes
\[
\Ll_{\rm enc}(G)=\frac12\E\|Gz-b\|_2^2.
\]
Since
\[
\nabla\Ll_{\rm enc}(G)=(G-G_\star)\Sigma_z,
\]
population gradient descent with step size \(\eta\) satisfies
\[
\boxed{
G_s-G_\star=(G_0-G_\star)(I-\eta\Sigma_z)^s.
}
\]

\begin{theorem}[Encoder optimization curve]
If \(0<\eta<2/\lambda_{\max}(\Sigma_z)\), then
\[
R_{\rm enc}(s)
=\frac1K\E\|G_sz-b\|_2^2
=\sigma^2+\frac1K\tr\!\left[
(G_s-G_\star)\Sigma_z(G_s-G_\star)^\T
\right],
\]
and
\[
R_{\rm enc}(s)-\sigma^2
\le
\|I-\eta\Sigma_z\|_{\op}^{2s}
\frac1K\tr\!\left[(G_0-G_\star)\Sigma_z(G_0-G_\star)^\T\right].
\]
\end{theorem}

\begin{proof}
The gradient formula gives the linear recursion above.  Insert it into the
quadratic population risk and diagonalize \(\Sigma_z\).
\end{proof}

\subsection{Encoder generalization}

For a held-out sample of \(n\) tasks and clipped loss
\[
\ell_c=\min(\ell,c),\qquad 0\le \ell_c\le c,
\]
the empirical Bernstein certificate used throughout is
\[
\boxed{
R_c
\le
\widehat R_c
+\sqrt{\frac{2\widehat V_c\log(2/\delta)}{n}}
+\frac{7c\log(2/\delta)}{3(n-1)} .
}
\]
In the Gaussian encoder model the exact population curve above is stronger
than a bound.  The experiment verifies
\[
\widehat R_{\rm enc}(800)=4.0020466789\times10^{-4},
\qquad
R_{\rm enc}(800)=4.0000000000\times10^{-4},
\]
\[
\widehat R_{\rm enc}(800)/R_{\rm enc}(800)=1.0005116697.
\]

\begin{figure}[H]
\centering
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_encoder_online_final.png}
\caption{Encoder online curve.}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{online_encoder_lowrank_recovery_final_plot.png}
\caption{Final encoder checkpoint.}
\end{subfigure}
\caption{Encoder-only risk is predicted by the exact population trajectory.}
\end{figure}

\section{Decoder-only theory}

\subsection{Frozen decoder is Richardson}

For one task, let the query system be
\[
Hx_\star=d,\qquad H\succ0.
\]
At frozen pretrained parameters \(s\), the decoder applies
\[
x_{\ell+1}=x_\ell+P_s(d-Hx_\ell).
\]
With \(e_\ell=x_\ell-x_\star\),
\[
\boxed{
e_{\ell+1}=(I-P_sH)e_\ell,
\qquad
e_L=(I-P_sH)^Le_0.
}
\]
Consequently
\[
\|e_L\|_2\le\|I-P_sH\|_2^L\|e_0\|_2.
\]
The exact depth risk for source covariance \(S=\E[x_\star x_\star^\T]\) is
\[
R_L(P_s)
=\frac1K\tr\left[(I-P_sH)^L S (I-P_sH)^{L\T}\right].
\]

\subsection{Linear-attention pretraining as preconditioner learning}

The free linear-attention decoder model is
\[
\widehat x=Pd,\qquad
\Ll_{\rm dec}(P)=\frac1{2K}\E\|Pd-x_\star\|_2^2.
\]
Let
\[
C=\E[dd^\T],
\qquad
B=\E[x_\star d^\T].
\]
Then
\[
\nabla\Ll_{\rm dec}(P)=\frac1K(PC-B),
\qquad
P_\star=BC^\dagger.
\]
With the rescaled gradient step
\[
P_{s+1}=P_s-\eta( P_sC-B),
\]
we get
\[
\boxed{
P_s-P_\star=(P_0-P_\star)(I-\eta C)^s.
}
\]
Thus the outer training curve is again a closed covariance-filtered curve.

\begin{figure}[H]
\centering
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_decoder_online_final.png}
\caption{Decoder online pretraining.}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_decoder_only_richardson_overlap.png}
\caption{Richardson overlap diagnostics.}
\end{subfigure}
\caption{Decoder-only training learns a preconditioner; evaluation at frozen
\(P_s\) is exactly Richardson.}
\end{figure}

\subsection{Decoder verification}

At the final decoder-only Flexformer checkpoint:
\[
\widehat R_{\rm dec}=7.674719\times10^{-3},
\qquad
U_{\rm EB}=1.771005\times10^{-1},
\]
\[
\Pr\{\|e_L\|_2>\|I-P_\theta H\|_2^L\|e_0\|_2\}=0.
\]
The error/bound ratio is \(2.2833\times10^{-2}\), the relative distance to the
oracle \(P_\star\) is \(2.6788\times10^{-1}\), and the stable rank of
\(P_\theta\) is \(2.9198\).

\begin{figure}[H]
\centering
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{decoder_only_flexformer_bounds.png}
\caption{Decoder generalization and bound.}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_preconditioner_training_strategy.png}
\caption{Preconditioner strategy.}
\end{subfigure}
\caption{The decoder value is not that it beats an exact direct solver.  Its
value is that pretraining learns a reusable preconditioner that lowers the
number of Richardson-like layers/iterations needed on new prompts.}
\end{figure}

\section{Encoder--decoder composition}

\subsection{Error decomposition}

Let \(\widehat z\) be the encoder output, and let
\[
x^\dagger(\widehat z)=H(\widehat z)^{-1}d(\widehat z)
\]
be the exact solution of the encoded query system.  The model output after
\(L\) decoder layers is \(\widehat x_L\).  Then
\[
\widehat x_L-x_\star
=
\underbrace{\widehat x_L-x^\dagger(\widehat z)}_{\rm decoder}
+
\underbrace{x^\dagger(\widehat z)-x_\star}_{\rm encoder}.
\]
Hence
\[
\|\widehat x_L-x_\star\|_2^2
\le
2\|\widehat x_L-x^\dagger(\widehat z)\|_2^2
+2\|x^\dagger(\widehat z)-x_\star\|_2^2.
\]
The first term is exact Richardson:
\[
\widehat x_L-x^\dagger(\widehat z)
=(I-P_\theta H(\widehat z))^L
(\widehat x_0-x^\dagger(\widehat z)).
\]

\subsection{Perturbation from latent error}

Assume
\[
H(z)=H_0+\sum_{k=1}^r z_kH_k,\qquad
d(z)=d_0+\sum_{k=1}^r z_kd_k,
\]
and \(H(z)\succeq \alpha I\) in a neighborhood of \(z\).  Then
\[
x^\dagger(z)=H(z)^{-1}d(z).
\]
For \(\Delta z=\widehat z-z\),
\[
x^\dagger(\widehat z)-x^\dagger(z)
=H(z)^{-1}\left[
\sum_k \Delta z_k(d_k-H_kx^\dagger(z))
\right]
+O(\|\Delta z\|^2).
\]
Thus, locally,
\[
\|x^\dagger(\widehat z)-x_\star\|_2
\le
\frac{1}{\alpha}
\left\|
\sum_k \Delta z_k(d_k-H_kx_\star)
\right\|_2
+O(\|\Delta z\|^2).
\]

\subsection{Stop-gradient triangular dynamics}

If the decoder is stopped from backpropagating into the encoder and the encoder
has its own loss, then the coupled training is triangular:
\[
G_{s+1}-G_\star=(G_s-G_\star)(I-\eta_e\Sigma_z),
\]
\[
P_{s+1}-P_\star(G_s)
=(P_s-P_\star(G_s))(I-\eta_d C(G_s))
+\bigl(P_\star(G_s)-P_\star(G_{s+1})\bigr).
\]
If the decoder is fast relative to the encoder, \(P_s\) tracks the slowly
moving \(P_\star(G_s)\).  If the decoder starts before the encoder has signal,
the first iterations train on noisy or poorly identified systems; this is the
transient seen in joint runs.

\begin{figure}[H]
\centering
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_encdec_online_final.png}
\caption{Online encoder--decoder.}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_encoder_decoder_richardson_overlap.png}
\caption{Composed overlap dynamics.}
\end{subfigure}
\caption{With stop-gradient and explicit encoder loss, the dynamics separate
into encoder recovery plus decoder preconditioner adaptation.}
\end{figure}

\begin{figure}[H]
\centering
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{encoder_decoder_flexformer_bounds.png}
\caption{Flexformer encoder--decoder certificate.}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{online_lowrank_encoder_decoder_final_plot.png}
\caption{Low-rank encoder--decoder final run.}
\end{subfigure}
\caption{Encoder--decoder experiments verify the identity decomposition and
zero pointwise Richardson-bound violations in the reported runs.}
\end{figure}

\section{Exact correspondence with low-rank linear-attention ICL}

\subsection{Paper model}

The low-rank ICL replica framework studies tasks
\[
w^\mu=\sqrt{D/r}\,A v^\mu,\qquad v^\mu\sim\N(0,I_r),
\]
and prompt pairs
\[
y_i^\mu=\langle w^\mu,x_i^\mu\rangle+\eps_i^\mu.
\]
The one-layer linear-attention architecture has the form
\[
\operatorname{Att}_\Theta(C)
=C+\frac1LVC(KC)^\T(QC),
\]
with a linear readout.  The resulting query prediction can be written as a
linear predictor on a prompt-derived feature matrix \(H^\mu\):
\[
\widehat y^\mu=\tr(W H^{\mu\T}).
\]

\subsection{Our exact mapping}

Our low-rank linear regression task is the same model after renaming:
\[
w^\mu\longleftrightarrow \beta(z^\mu)=U_\star z^\mu,
\qquad
A\longleftrightarrow U_\star,
\qquad
v^\mu\longleftrightarrow z^\mu.
\]
For PDE inverse problems, the task vector is not directly a regression weight;
it is the coefficient vector \(z\) in the weak system:
\[
G(\Pi_z)z=b(\Pi_z).
\]
After Galerkin testing, this is still a low-rank linear task.  The context
length in the ICL paper corresponds to the number of weak prompt equations:
\[
L_{\rm paper}\longleftrightarrow n=mq.
\]
The task rank is
\[
r_{\rm paper}\longleftrightarrow \rank(\Sigma_z).
\]
The pretraining task diversity \(M_0\) in the paper corresponds to the number
of distinct latent directions/tasks covered by pretraining, i.e. the empirical
rank and conditioning of the sampled \(z\)-cloud.

\subsection{Decoder as the algorithmic component}

The paper decomposes prediction into an algorithmic component and a noise
component.  In our notation, the algorithmic component is the learned
preconditioner/solver:
\[
x_{\ell+1}=x_\ell+P_\theta(d-Hx_\ell).
\]
The noise component is any part of the learned map not aligned with the prompt
least-squares geometry.  In the exact linear-attention/free-\(P\) reduction,
pretraining drives
\[
P_s\to P_\star=BC^\dagger,
\]
so the algorithmic component is precisely the covariance-optimal linear
preconditioner.

\subsection{Flexformer and learnable attention kernels}

For kernelized linear attention,
\[
A_\theta(i,j)
=\frac{k_\omega(q_i,k_j)}{\sum_a k_\omega(q_i,k_a)},
\qquad
P_\theta=D_aA_\theta D_v+D_s.
\]
For fixed \(\theta\), all decoder certificates remain exact because they only
use \(P_\theta\).  For training \(\theta\), the finite-set recursion is exact:
\[
z_{i,\ell+1}=z_{i,\ell}+P_\theta r_{i,\ell},
\qquad
r_{i,\ell}=d_i-H_iz_{i,\ell},
\]
\[
\lambda_{i,L}=\frac{z_{i,L}-z_i^\star}{NK},
\qquad
\lambda_{i,\ell}=(I-P_\theta H_i)^\T\lambda_{i,\ell+1},
\]
\[
\nabla_P\Ll_N(P_\theta)
=\sum_{i=1}^N\sum_{\ell=0}^{L-1}\lambda_{i,\ell+1}r_{i,\ell}^\T,
\]
\[
\theta_{s+1}
=\theta_s-\eta\,DP_{\theta_s}^{*}
\left[\nabla_P\Ll_N(P_{\theta_s})\right].
\]
The checked relative errors were
\[
\operatorname{relerr}(\nabla_P)=1.727\times10^{-16},
\qquad
\operatorname{relerr}(\nabla_\theta)=1.048\times10^{-15}.
\]
What is not proved is a global scalar replica closure for this nonlinear
outer-training law.

\begin{figure}[H]
\centering
\begin{subfigure}{0.32\linewidth}
\includegraphics[width=\linewidth]{exact_flexformer_gradient_dynamics.png}
\caption{Exact finite-set gradients.}
\end{subfigure}
\hfill
\begin{subfigure}{0.32\linewidth}
\includegraphics[width=\linewidth]{flexformer_decoder_ntk_training_curve.png}
\caption{Decoder NTK local curve.}
\end{subfigure}
\hfill
\begin{subfigure}{0.32\linewidth}
\includegraphics[width=\linewidth]{flexformer_gp_ntk_training_curve.png}
\caption{GP/KRR NTK local curve.}
\end{subfigure}
\caption{For nonlinear or learnable-kernel attention, the finite gradient law is
exact and local NTK curves can be accurate.  The global replica closure remains
an order-parameter problem.}
\end{figure}

\section{Replica order parameters}

\subsection{Quenched free energy}

For a training set \(\Dd_N\), define a Gibbs posterior over effective attention
parameters \(P\):
\[
Z_N(\beta)
=\int\exp\{-\beta N\widehat\Rr_N(P)\}\,d\pi(P).
\]
The quenched free energy is
\[
\Ff(\beta)
=-\frac1{\beta N}\E_{\Dd_N}\log Z_N(\beta).
\]
The replica identity is
\[
\E\log Z_N
=\lim_{n\to0}\frac{\E Z_N^n-1}{n}.
\]
For \(n\) replicas \(P^1,\dots,P^n\), the calculation closes only after
introducing overlaps.

\subsection{Which overlaps are needed}

Spectra alone are not enough.  The decoder risk is
\[
R_L(P)
=\frac1K\tr\left[(I-PH)^LS(I-PH)^{L\T}\right].
\]
It depends on the relative eigenvectors of \(P\), \(H\), and \(S\), not merely
on eigenvalues of \(P\).  Required order parameters include
\[
Q_{ab}=\frac1K\tr(P^aP^{b\T}),
\qquad
M_a=\frac1K\tr(P^aH),
\]
\[
T_{ab}^{(1)}=\frac1K\tr(P^aHP^bH),
\qquad
T_{ab}^{(S)}=\frac1K\tr(P^aHSH^\T P^{b\T}).
\]
For encoder recovery, the corresponding overlaps are
\[
Q^G_{ab}=\frac1r\tr\!\left[(G^a-G_\star)\Sigma_z(G^b-G_\star)^\T\right].
\]
For the prompt spectrum, the scalar MP law supplies the marginal spectral
measure of \(G_m^\T G_m\); the replica calculation supplies how the learned
attention aligns to it.

\begin{figure}[H]
\centering
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{spectral_closure_obstruction.png}
\caption{Same spectra, different risk.}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{joint_order_parameter_dynamics.png}
\caption{Logged overlap state.}
\end{subfigure}
\caption{The obstruction experiment fixes eigenvalues and singular values but
changes eigenbasis alignment.  Depth-1 risks differ by \(1.541\times10^{-33}\)
versus \(1.090\times10^{-1}\).  The logged joint order parameters reconstruct
checkpoint risks with max absolute error \(1.137\times10^{-12}\).}
\end{figure}

\subsection{Replica diagnostic plots}

\begin{figure}[H]
\centering
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_replica_dynamic_order_parameters.png}
\caption{Dynamic order parameters.}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_replica_top_spectra_grid.png}
\caption{Top spectra grid.}
\end{subfigure}

\vspace{0.7em}
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_replica_latent_prompt_spectral_window.png}
\caption{Latent prompt spectral window.}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_qk_lowrank_dynamics.png}
\caption{Q/K low-rank dynamics.}
\end{subfigure}
\caption{Replica-style diagnostics: prompt spectra, low-rank Q/K motion, and
overlap variables.  These are the quantities a closed dynamical replica theory
must predict.}
\end{figure}

\section{Scaling laws}

\subsection{Prompt size}

Let \(n=mq\) be the number of weak equations.  In the whitened Gaussian prompt
model with OLS and noise variance \(\sigma^2\),
\[
\E\|\widehat z-z\|_2^2
=\sigma^2\,\E\tr\left[(G_m^\T G_m)^{-1}\right]
=\sigma^2\frac{r}{n-r-1},
\qquad n>r+1.
\]
With ridge \(\lambda\),
\[
R_z(\lambda)
=\E\tr\left[
\lambda^2(G_m^\T G_m+\lambda I)^{-2}\Sigma_z
+\sigma^2G_m^\T G_m(G_m^\T G_m+\lambda I)^{-2}
\right].
\]
In the proportional limit this becomes an MP integral.

\subsection{Training task count}

Let \(N\) be the number of pretraining tasks.  For clipped loss, the empirical
Bernstein deviation is
\[
O\!\left(\sqrt{\frac{\widehat V\log(1/\delta)}{N}}
+\frac{c\log(1/\delta)}{N}\right).
\]
For linear free-\(P\) pretraining with effective covariance \(C\), optimization
error at outer time \(s\) is exactly
\[
\Rr_{\rm dec}(P_s)-\Rr_{\rm dec}(P_\star)
=\frac1K\tr\{(P_0-P_\star)(I-\eta C)^s
C(I-\eta C)^s(P_0-P_\star)^\T\}.
\]
Finite \(N\) affects \(C,B\) through sample fluctuations; this is the finite
pretraining-data regularization captured by the low-rank replica theory.

\subsection{Task rank and diversity}

The task rank \(r\) appears in three places:
\[
\gamma=\frac{r}{mq},
\qquad
d_{\rm task}=\frac{\tr(\Sigma_z)^2}{\tr(\Sigma_z^2)},
\qquad
\rank(U_\star)=r.
\]
Generalization improves when \(mq\gg r\), when \(N\) covers the effective
latent support, and when the decoder preconditioner aligns with the stiff
eigenvectors of the query Hessians.

\begin{figure}[H]
\centering
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{certified_k4_scaling.png}
\caption{Certified K=4 scaling sweep.}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_generalization_gap_deff_over_n.png}
\caption{Effective dimension over \(N\).}
\end{subfigure}
\caption{Scaling evidence.  The certified sweep has 96 rows and max reported
violation \(0\).}
\end{figure}

\section{Optimization and generalization plots}

\begin{figure}[H]
\centering
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_optimization_contraction_fit.png}
\caption{Measured vs predicted contraction.}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_optimization_contraction_factors.png}
\caption{Contraction factors.}
\end{subfigure}

\vspace{0.7em}
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_generalization_train_eval_fit.png}
\caption{Train/eval risk fit.}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_risk_prediction_overlap.png}
\caption{Risk prediction overlap.}
\end{subfigure}
\caption{The deterministic optimization curves track contraction factors; the
generalization plots compare train and held-out behavior.}
\end{figure}

\begin{figure}[H]
\centering
\begin{subfigure}{0.32\linewidth}
\includegraphics[width=\linewidth]{fig_scalar_decoder.png}
\caption{Decoder scalar law.}
\end{subfigure}
\hfill
\begin{subfigure}{0.32\linewidth}
\includegraphics[width=\linewidth]{fig_scalar_encoder.png}
\caption{Encoder scalar law.}
\end{subfigure}
\hfill
\begin{subfigure}{0.32\linewidth}
\includegraphics[width=\linewidth]{fig_scalar_coupled.png}
\caption{Coupled scalar law.}
\end{subfigure}
\caption{Scalar reductions are exact for the deliberately scalarized systems.
They explain the shape of the curves but do not replace the overlap theory for
full matrices.}
\end{figure}

\section{What is proved and what is not}

\begin{center}
\begin{tabular}{p{0.33\linewidth}p{0.55\linewidth}}
\toprule
Claim & Status \\
\midrule
PDE prompt \(\to G_mz=b_m\) & Exact after Galerkin/weak-form discretization.\\
Finite-rank GP \(\leftrightarrow\) KRR & Exact primal--dual identity.\\
Encoder-only optimization & Exact population GD curve.\\
Decoder frozen optimization & Exact Richardson identity for every \(P_\theta\).\\
Encoder--decoder with stop-gradient & Exact triangular decomposition plus perturbation bound.\\
Held-out generalization & Certified by empirical Bernstein after clipping; exact Gaussian risk in encoder model.\\
Linear-attention low-rank ICL correspondence & Exact after finite least-squares recast and linear attention/readout reduction.\\
Softmax/Flexformer global training theory & Exact finite recursion and local NTK diagnostics; no proved global scalar replica closure yet.\\
Replica closure variables & Must include spectra and overlaps; spectra alone are false.\\
\bottomrule
\end{tabular}
\end{center}

\section{Conclusion}

The rigorous core is finite-dimensional.  The encoder solves a prompt inverse
problem; the decoder learns and applies a preconditioner; the composition
separates into identification error and solver error.  Linear attention fits
the low-rank ICL replica framework exactly once the prompt is written as a
least-squares feature matrix.  The difficult remaining theory is not the frozen
optimization bound--that is exact--but the global outer-training law for
nonlinear learnable attention kernels.  The experiments show which macroscopic
quantities that law must track: prompt MP spectra, Q/K low-rank motion, and
overlaps among \(P\), \(H\), and \(S\).

\appendix

\section{Figure index}

All figures below are included so the paper is self-contained as an audit
artifact.

\begin{figure}[H]
\centering
\includegraphics[width=0.7\linewidth]{fig_scalar_counterexample.png}
\caption{Scalar counterexample: scalar dynamics do not imply a full matrix
closure.}
\end{figure}

\begin{figure}[H]
\centering
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_replica_prompt_aspect_scaling.png}
\caption{Replica prompt aspect scaling.}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{fig_replica_latent_prompt_spectral_window.png}
\caption{Latent prompt spectral window.}
\end{subfigure}
\caption{Additional prompt spectral diagnostics.}
\end{figure}

\begin{figure}[H]
\centering
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{online_encoder_lowrank_recovery_final_plot.png}
\caption{Encoder final plot.}
\end{subfigure}
\hfill
\begin{subfigure}{0.48\linewidth}
\includegraphics[width=\linewidth]{online_lowrank_encoder_decoder_final_plot.png}
\caption{Encoder--decoder final plot.}
\end{subfigure}
\caption{Final online runs copied from the optimization/generalization report.}
\end{figure}

\section{References}

\begin{thebibliography}{9}

\bibitem{lowrank-icl-replica}
K. Takanami, T. Takahashi, and Y. Kabashima.
\newblock Learning Linear Regression with Low-Rank Tasks In-Context.
\newblock arXiv:2510.04548, 2026.
\newblock \url{https://arxiv.org/abs/2510.04548}.

\bibitem{linear-attn-dynamics}
Y. Zhang, A. K. Singh, P. E. Latham, and A. Saxe.
\newblock Training Dynamics of In-Context Learning in Linear Attention.
\newblock arXiv:2501.16265, 2025.
\newblock \url{https://arxiv.org/abs/2501.16265}.

\bibitem{flexformer}
H. Zhang and F. Zhou.
\newblock Flexformer: Flexible Linear Transformer with Learnable Attention Kernel.
\newblock arXiv:2606.27748, 2026.
\newblock \url{https://arxiv.org/abs/2606.27748}.

\end{thebibliography}

\end{document}
