RELISH: LLM REgression with a Latent Iterative State Head. Su, Y. & Lease, M. In Proceedings of the 3rd Conference on Language Modeling (COLM), San Franciso, CA, October, 2026. arXiv:2604.01206 [cs.CL]![RELISH: LLM REgression with a Latent Iterative State Head [link]](https://bibbase.org/img/filetypes/link.svg "link")
Paper doi abstract bibtex We present RELISH (REgression with a Latent Iterative State Head), a novel, lightweight architecture designed for text regression with large language models. Rather than decoding numeric targets as text or aggregating multiple generated outputs, RELISH predicts scalar values directly from frozen LLM representations by iteratively refining a learned latent state through cross-attention over token-level representations, and then mapping the final state to a point estimate with a linear regressor. Across five datasets, four LLM backbones, and two LLM training regimes, RELISH consistently outperforms prior baselines from all three major LLM regression families, including autoregressive decoding, regression-aware inference, and existing predictive head methods. Despite these gains, RELISH remains highly parameter-efficient, requiring only ∼3.4–3.7M trainable parameters across frozen LLM backbones (only 0.01–0.04% additional overhead), far less than LoRA-based alternatives that grow with model size (0.26–0.42%).
@inproceedings{su_relish_2026,
address = {San Franciso, CA},
title = {{RELISH}: {LLM} {REgression} with a {Latent} {Iterative} {State} {Head}},
url = {http://arxiv.org/abs/2604.01206},
doi = {10.48550/arXiv.2604.01206},
abstract = {We present RELISH (REgression with a Latent Iterative State Head), a novel, lightweight architecture designed for text regression with large language models. Rather than decoding numeric targets as text or aggregating multiple generated outputs, RELISH predicts scalar values directly from frozen LLM representations by iteratively refining a learned latent state through cross-attention over token-level representations, and then mapping the final state to a point estimate with a linear regressor. Across five datasets, four LLM backbones, and two LLM training regimes, RELISH consistently outperforms prior baselines from all three major LLM regression families, including autoregressive decoding, regression-aware inference, and existing predictive head methods. Despite these gains, RELISH remains highly parameter-efficient, requiring only ∼3.4–3.7M trainable parameters across frozen LLM backbones (only 0.01–0.04\% additional overhead), far less than LoRA-based alternatives that grow with model size (0.26–0.42\%).},
language = {en},
urldate = {2026-07-09},
booktitle = {Proceedings of the 3rd {Conference} on {Language} {Modeling} ({COLM})},
author = {Su, Yiheng and Lease, Matthew},
month = oct,
year = {2026},
note = {arXiv:2604.01206 [cs.CL]},
keywords = {Computer Science - Computation and Language, Computer Science - Machine Learning, WG: Explorable},
}
Downloads: 0
{"_id":"dtDLLPKjoSBifoJxj","bibbaseid":"su-lease-relishllmregressionwithalatentiterativestatehead-2026","author_short":["Su, Y.","Lease, M."],"bibdata":{"bibtype":"inproceedings","type":"inproceedings","address":"San Franciso, CA","title":"RELISH: LLM REgression with a Latent Iterative State Head","url":"http://arxiv.org/abs/2604.01206","doi":"10.48550/arXiv.2604.01206","abstract":"We present RELISH (REgression with a Latent Iterative State Head), a novel, lightweight architecture designed for text regression with large language models. Rather than decoding numeric targets as text or aggregating multiple generated outputs, RELISH predicts scalar values directly from frozen LLM representations by iteratively refining a learned latent state through cross-attention over token-level representations, and then mapping the final state to a point estimate with a linear regressor. Across five datasets, four LLM backbones, and two LLM training regimes, RELISH consistently outperforms prior baselines from all three major LLM regression families, including autoregressive decoding, regression-aware inference, and existing predictive head methods. Despite these gains, RELISH remains highly parameter-efficient, requiring only ∼3.4–3.7M trainable parameters across frozen LLM backbones (only 0.01–0.04% additional overhead), far less than LoRA-based alternatives that grow with model size (0.26–0.42%).","language":"en","urldate":"2026-07-09","booktitle":"Proceedings of the 3rd Conference on Language Modeling (COLM)","author":[{"propositions":[],"lastnames":["Su"],"firstnames":["Yiheng"],"suffixes":[]},{"propositions":[],"lastnames":["Lease"],"firstnames":["Matthew"],"suffixes":[]}],"month":"October","year":"2026","note":"arXiv:2604.01206 [cs.CL]","keywords":"Computer Science - Computation and Language, Computer Science - Machine Learning, WG: Explorable","bibtex":"@inproceedings{su_relish_2026,\n\taddress = {San Franciso, CA},\n\ttitle = {{RELISH}: {LLM} {REgression} with a {Latent} {Iterative} {State} {Head}},\n\turl = {http://arxiv.org/abs/2604.01206},\n\tdoi = {10.48550/arXiv.2604.01206},\n\tabstract = {We present RELISH (REgression with a Latent Iterative State Head), a novel, lightweight architecture designed for text regression with large language models. Rather than decoding numeric targets as text or aggregating multiple generated outputs, RELISH predicts scalar values directly from frozen LLM representations by iteratively refining a learned latent state through cross-attention over token-level representations, and then mapping the final state to a point estimate with a linear regressor. Across five datasets, four LLM backbones, and two LLM training regimes, RELISH consistently outperforms prior baselines from all three major LLM regression families, including autoregressive decoding, regression-aware inference, and existing predictive head methods. Despite these gains, RELISH remains highly parameter-efficient, requiring only ∼3.4–3.7M trainable parameters across frozen LLM backbones (only 0.01–0.04\\% additional overhead), far less than LoRA-based alternatives that grow with model size (0.26–0.42\\%).},\n\tlanguage = {en},\n\turldate = {2026-07-09},\n\tbooktitle = {Proceedings of the 3rd {Conference} on {Language} {Modeling} ({COLM})},\n\tauthor = {Su, Yiheng and Lease, Matthew},\n\tmonth = oct,\n\tyear = {2026},\n\tnote = {arXiv:2604.01206 [cs.CL]},\n\tkeywords = {Computer Science - Computation and Language, Computer Science - Machine Learning, WG: Explorable},\n}\n\n\n\n","author_short":["Su, Y.","Lease, M."],"key":"su_relish_2026","id":"su_relish_2026","bibbaseid":"su-lease-relishllmregressionwithalatentiterativestatehead-2026","role":"author","urls":{"Paper":"http://arxiv.org/abs/2604.01206"},"keyword":["Computer Science - Computation and Language","Computer Science - Machine Learning","WG: Explorable"],"metadata":{"authorlinks":{}},"downloads":0},"bibtype":"inproceedings","biburl":"https://bibbase.org/zotero-group/pratikmhatre/5933976","dataSources":["yJr5AAtJ5Sz3Q4WT4"],"keywords":["computer science - computation and language","computer science - machine learning","wg: explorable"],"search_terms":["relish","llm","regression","latent","iterative","state","head","su","lease"],"title":"RELISH: LLM REgression with a Latent Iterative State Head","year":2026}