RELISH: LLM REgression with a Latent Iterative State Head

RELISH: LLM REgression with a Latent Iterative State Head. Su, Y. & Lease, M. In Proceedings of the 3rd Conference on Language Modeling (COLM), San Franciso, CA, October, 2026.

Paper doi abstract bibtex

We present RELISH (REgression with a Latent Iterative State Head), a novel, lightweight architecture designed for text regression with large language models. Rather than decoding numeric targets as text or aggregating multiple generated outputs, RELISH predicts scalar values directly from frozen LLM representations by iteratively refining a learned latent state through cross-attention over token-level representations, and then mapping the final state to a point estimate with a linear regressor. Across six datasets, four LLM backbones, and two LLM training regimes, RELISH consistently outperforms prior baselines from all three major LLM regression families, including autoregressive decoding, regression-aware inference, and existing predictive head methods. Despite these gains, RELISH remains highly parameter-efficient, requiring only ∼3.4–3.7M trainable parameters across frozen LLM backbones (only 0.01–0.04% additional overhead), far less than LoRA-based alternatives that grow with model size (0.26–0.42%). Our code is available at https://github.com/SamSoup/RELISH.

@inproceedings{su_relish_2026,
	address = {San Franciso, CA},
	title = {{RELISH}: {LLM} {REgression} with a {Latent} {Iterative} {State} {Head}},
	url = {http://arxiv.org/abs/2604.01206},
	doi = {10.48550/arXiv.2604.01206},
	abstract = {We present RELISH (REgression with a Latent Iterative State Head), a novel, lightweight architecture designed for text regression with large language models. Rather than decoding numeric targets as text or aggregating multiple generated outputs, RELISH predicts scalar values directly from frozen LLM representations by iteratively refining a learned latent state through cross-attention over token-level representations, and then mapping the final state to a point estimate with a linear regressor. Across six datasets, four LLM backbones, and two LLM training regimes, RELISH consistently outperforms prior baselines from all three major LLM regression families, including autoregressive decoding, regression-aware inference, and existing predictive head methods. Despite these gains, RELISH remains highly parameter-efficient, requiring only ∼3.4–3.7M trainable parameters across frozen LLM backbones (only 0.01–0.04\% additional overhead), far less than LoRA-based alternatives that grow with model size (0.26–0.42\%). Our code is available at https://github.com/SamSoup/RELISH.},
	language = {en},
	booktitle = {Proceedings of the 3rd {Conference} on {Language} {Modeling} ({COLM})},
	author = {Su, Yiheng and Lease, Matthew},
	month = oct,
	year = {2026},
}

Downloads: 0

{"_id":"dtDLLPKjoSBifoJxj","bibbaseid":"su-lease-relishllmregressionwithalatentiterativestatehead-2026","author_short":["Su, Y.","Lease, M."],"bibdata":{"bibtype":"inproceedings","type":"inproceedings","address":"San Franciso, CA","title":"RELISH: LLM REgression with a Latent Iterative State Head","url":"http://arxiv.org/abs/2604.01206","doi":"10.48550/arXiv.2604.01206","abstract":"We present RELISH (REgression with a Latent Iterative State Head), a novel, lightweight architecture designed for text regression with large language models. Rather than decoding numeric targets as text or aggregating multiple generated outputs, RELISH predicts scalar values directly from frozen LLM representations by iteratively refining a learned latent state through cross-attention over token-level representations, and then mapping the final state to a point estimate with a linear regressor. Across six datasets, four LLM backbones, and two LLM training regimes, RELISH consistently outperforms prior baselines from all three major LLM regression families, including autoregressive decoding, regression-aware inference, and existing predictive head methods. Despite these gains, RELISH remains highly parameter-efficient, requiring only ∼3.4–3.7M trainable parameters across frozen LLM backbones (only 0.01–0.04% additional overhead), far less than LoRA-based alternatives that grow with model size (0.26–0.42%). Our code is available at https://github.com/SamSoup/RELISH.","language":"en","booktitle":"Proceedings of the 3rd Conference on Language Modeling (COLM)","author":[{"propositions":[],"lastnames":["Su"],"firstnames":["Yiheng"],"suffixes":[]},{"propositions":[],"lastnames":["Lease"],"firstnames":["Matthew"],"suffixes":[]}],"month":"October","year":"2026","bibtex":"@inproceedings{su_relish_2026,\n\taddress = {San Franciso, CA},\n\ttitle = {{RELISH}: {LLM} {REgression} with a {Latent} {Iterative} {State} {Head}},\n\turl = {http://arxiv.org/abs/2604.01206},\n\tdoi = {10.48550/arXiv.2604.01206},\n\tabstract = {We present RELISH (REgression with a Latent Iterative State Head), a novel, lightweight architecture designed for text regression with large language models. Rather than decoding numeric targets as text or aggregating multiple generated outputs, RELISH predicts scalar values directly from frozen LLM representations by iteratively refining a learned latent state through cross-attention over token-level representations, and then mapping the final state to a point estimate with a linear regressor. Across six datasets, four LLM backbones, and two LLM training regimes, RELISH consistently outperforms prior baselines from all three major LLM regression families, including autoregressive decoding, regression-aware inference, and existing predictive head methods. Despite these gains, RELISH remains highly parameter-efficient, requiring only ∼3.4–3.7M trainable parameters across frozen LLM backbones (only 0.01–0.04\\% additional overhead), far less than LoRA-based alternatives that grow with model size (0.26–0.42\\%). Our code is available at https://github.com/SamSoup/RELISH.},\n\tlanguage = {en},\n\tbooktitle = {Proceedings of the 3rd {Conference} on {Language} {Modeling} ({COLM})},\n\tauthor = {Su, Yiheng and Lease, Matthew},\n\tmonth = oct,\n\tyear = {2026},\n}\n\n\n\n\n\n\n\n","author_short":["Su, Y.","Lease, M."],"key":"su_relish_2026","id":"su_relish_2026","bibbaseid":"su-lease-relishllmregressionwithalatentiterativestatehead-2026","role":"author","urls":{"Paper":"http://arxiv.org/abs/2604.01206"},"metadata":{"authorlinks":{}}},"bibtype":"inproceedings","biburl":"https://bibbase.org/zotero-group/pratikmhatre/5933976","dataSources":["yJr5AAtJ5Sz3Q4WT4"],"keywords":[],"search_terms":["relish","llm","regression","latent","iterative","state","head","su","lease"],"title":"RELISH: LLM REgression with a Latent Iterative State Head","year":2026}