PEACE: Preemptive and Efficient Cluster Scheduling for LLM Inference with Mixed Prompts. Zhang, Z. & Shen, H. In 2026 IEEE International Parallel and Distributed Processing Symposium (IPDPS), pages 1272–1286, New Orleans, LA, USA, May, 2026. IEEE. ![PEACE: Preemptive and Efficient Cluster Scheduling for LLM Inference with Mixed Prompts [link]](https://bibbase.org/img/filetypes/link.svg "link")
Paper doi abstract bibtex The scaling of transformer-based Large Language Models (LLMs) has significantly expanded their context lengths, enabling applications where inputs exceed 100K tokens. Our analysis of a recent Azure LLM inference trace reveals a highly skewed long-tail distribution of input lengths, with approximately 10% of inputs longer than 4K tokens. Existing cluster-level LLM scheduling strategies, including First-In-First-Out (FIFO), reservation-based, and priority-based approaches, primarily target short-input requests with lengths below 4K and fail to address this heterogeneity, leading to inefficiencies such as headof-line blocking, resource underutilization, and starvation of longinput requests. We propose PEACE, a PreEmptive And efficient Cluster schEduling system for LLM inference, to improve the performance of short-input requests with minimal impact on the Job Completion Time (JCT) of long-input requests. PEACE introduces the following key techniques: 1) preemptive scheduling that prioritizes short-input requests for their performance; 2) coordinated prefill-decode colocation and disaggregation, which reduces both the duration and frequency of preemptions; 3) fast Sequence Parallelism (SP) that minimizes the prefill time of longinput requests to further reduce the likelihood and frequency of preemptions. Evaluations based on Azure LLM inference trace show that, compared to state-of-the-art cluster-level LLM inference schedulers, PEACE reduces the 99th percentile queueing delay of short-input requests by up to 92% and improves their throughput by up to 595%, without significantly affecting the JCT of long-input requests. We open-sourced our code.
@inproceedings{zhang_peace_2026,
address = {New Orleans, LA, USA},
title = {{PEACE}: {Preemptive} and {Efficient} {Cluster} {Scheduling} for {LLM} {Inference} with {Mixed} {Prompts}},
copyright = {https://doi.org/10.15223/policy-029},
isbn = {979-8-3195-0602-3},
shorttitle = {{PEACE}},
url = {https://ieeexplore.ieee.org/document/11575397/},
doi = {10.1109/IPDPS65963.2026.00105},
abstract = {The scaling of transformer-based Large Language Models (LLMs) has significantly expanded their context lengths, enabling applications where inputs exceed 100K tokens. Our analysis of a recent Azure LLM inference trace reveals a highly skewed long-tail distribution of input lengths, with approximately 10\% of inputs longer than 4K tokens. Existing cluster-level LLM scheduling strategies, including First-In-First-Out (FIFO), reservation-based, and priority-based approaches, primarily target short-input requests with lengths below 4K and fail to address this heterogeneity, leading to inefficiencies such as headof-line blocking, resource underutilization, and starvation of longinput requests. We propose PEACE, a PreEmptive And efficient Cluster schEduling system for LLM inference, to improve the performance of short-input requests with minimal impact on the Job Completion Time (JCT) of long-input requests. PEACE introduces the following key techniques: 1) preemptive scheduling that prioritizes short-input requests for their performance; 2) coordinated prefill-decode colocation and disaggregation, which reduces both the duration and frequency of preemptions; 3) fast Sequence Parallelism (SP) that minimizes the prefill time of longinput requests to further reduce the likelihood and frequency of preemptions. Evaluations based on Azure LLM inference trace show that, compared to state-of-the-art cluster-level LLM inference schedulers, PEACE reduces the 99th percentile queueing delay of short-input requests by up to 92\% and improves their throughput by up to 595\%, without significantly affecting the JCT of long-input requests. We open-sourced our code.},
language = {en},
urldate = {2026-07-07},
booktitle = {2026 {IEEE} {International} {Parallel} and {Distributed} {Processing} {Symposium} ({IPDPS})},
publisher = {IEEE},
author = {Zhang, Zeyu and Shen, Haiying},
month = may,
year = {2026},
keywords = {WG: Explorable},
pages = {1272--1286},
}
Downloads: 0
{"_id":"yqqG5KxLHneAAeyAM","bibbaseid":"zhang-shen-peacepreemptiveandefficientclusterschedulingforllminferencewithmixedprompts-2026","author_short":["Zhang, Z.","Shen, H."],"bibdata":{"bibtype":"inproceedings","type":"inproceedings","address":"New Orleans, LA, USA","title":"PEACE: Preemptive and Efficient Cluster Scheduling for LLM Inference with Mixed Prompts","copyright":"https://doi.org/10.15223/policy-029","isbn":"979-8-3195-0602-3","shorttitle":"PEACE","url":"https://ieeexplore.ieee.org/document/11575397/","doi":"10.1109/IPDPS65963.2026.00105","abstract":"The scaling of transformer-based Large Language Models (LLMs) has significantly expanded their context lengths, enabling applications where inputs exceed 100K tokens. Our analysis of a recent Azure LLM inference trace reveals a highly skewed long-tail distribution of input lengths, with approximately 10% of inputs longer than 4K tokens. Existing cluster-level LLM scheduling strategies, including First-In-First-Out (FIFO), reservation-based, and priority-based approaches, primarily target short-input requests with lengths below 4K and fail to address this heterogeneity, leading to inefficiencies such as headof-line blocking, resource underutilization, and starvation of longinput requests. We propose PEACE, a PreEmptive And efficient Cluster schEduling system for LLM inference, to improve the performance of short-input requests with minimal impact on the Job Completion Time (JCT) of long-input requests. PEACE introduces the following key techniques: 1) preemptive scheduling that prioritizes short-input requests for their performance; 2) coordinated prefill-decode colocation and disaggregation, which reduces both the duration and frequency of preemptions; 3) fast Sequence Parallelism (SP) that minimizes the prefill time of longinput requests to further reduce the likelihood and frequency of preemptions. Evaluations based on Azure LLM inference trace show that, compared to state-of-the-art cluster-level LLM inference schedulers, PEACE reduces the 99th percentile queueing delay of short-input requests by up to 92% and improves their throughput by up to 595%, without significantly affecting the JCT of long-input requests. We open-sourced our code.","language":"en","urldate":"2026-07-07","booktitle":"2026 IEEE International Parallel and Distributed Processing Symposium (IPDPS)","publisher":"IEEE","author":[{"propositions":[],"lastnames":["Zhang"],"firstnames":["Zeyu"],"suffixes":[]},{"propositions":[],"lastnames":["Shen"],"firstnames":["Haiying"],"suffixes":[]}],"month":"May","year":"2026","keywords":"WG: Explorable","pages":"1272–1286","bibtex":"@inproceedings{zhang_peace_2026,\n\taddress = {New Orleans, LA, USA},\n\ttitle = {{PEACE}: {Preemptive} and {Efficient} {Cluster} {Scheduling} for {LLM} {Inference} with {Mixed} {Prompts}},\n\tcopyright = {https://doi.org/10.15223/policy-029},\n\tisbn = {979-8-3195-0602-3},\n\tshorttitle = {{PEACE}},\n\turl = {https://ieeexplore.ieee.org/document/11575397/},\n\tdoi = {10.1109/IPDPS65963.2026.00105},\n\tabstract = {The scaling of transformer-based Large Language Models (LLMs) has significantly expanded their context lengths, enabling applications where inputs exceed 100K tokens. Our analysis of a recent Azure LLM inference trace reveals a highly skewed long-tail distribution of input lengths, with approximately 10\\% of inputs longer than 4K tokens. Existing cluster-level LLM scheduling strategies, including First-In-First-Out (FIFO), reservation-based, and priority-based approaches, primarily target short-input requests with lengths below 4K and fail to address this heterogeneity, leading to inefficiencies such as headof-line blocking, resource underutilization, and starvation of longinput requests. We propose PEACE, a PreEmptive And efficient Cluster schEduling system for LLM inference, to improve the performance of short-input requests with minimal impact on the Job Completion Time (JCT) of long-input requests. PEACE introduces the following key techniques: 1) preemptive scheduling that prioritizes short-input requests for their performance; 2) coordinated prefill-decode colocation and disaggregation, which reduces both the duration and frequency of preemptions; 3) fast Sequence Parallelism (SP) that minimizes the prefill time of longinput requests to further reduce the likelihood and frequency of preemptions. Evaluations based on Azure LLM inference trace show that, compared to state-of-the-art cluster-level LLM inference schedulers, PEACE reduces the 99th percentile queueing delay of short-input requests by up to 92\\% and improves their throughput by up to 595\\%, without significantly affecting the JCT of long-input requests. We open-sourced our code.},\n\tlanguage = {en},\n\turldate = {2026-07-07},\n\tbooktitle = {2026 {IEEE} {International} {Parallel} and {Distributed} {Processing} {Symposium} ({IPDPS})},\n\tpublisher = {IEEE},\n\tauthor = {Zhang, Zeyu and Shen, Haiying},\n\tmonth = may,\n\tyear = {2026},\n\tkeywords = {WG: Explorable},\n\tpages = {1272--1286},\n}\n\n\n\n","author_short":["Zhang, Z.","Shen, H."],"key":"zhang_peace_2026","id":"zhang_peace_2026","bibbaseid":"zhang-shen-peacepreemptiveandefficientclusterschedulingforllminferencewithmixedprompts-2026","role":"author","urls":{"Paper":"https://ieeexplore.ieee.org/document/11575397/"},"keyword":["WG: Explorable"],"metadata":{"authorlinks":{}},"downloads":0},"bibtype":"inproceedings","biburl":"https://bibbase.org/zotero-group/pratikmhatre/5933976","dataSources":["yJr5AAtJ5Sz3Q4WT4"],"keywords":["wg: explorable"],"search_terms":["peace","preemptive","efficient","cluster","scheduling","llm","inference","mixed","prompts","zhang","shen"],"title":"PEACE: Preemptive and Efficient Cluster Scheduling for LLM Inference with Mixed Prompts","year":2026}