@proceedings {422, title = {Chi: A Scalable and Programmable Control Plane for Distributed Stream Processing Systems}, journal = {44th International Conference on Very Large Data Bases (VLDB)}, volume = {10}, year = {2018}, month = {09/2018}, address = {Rio de Janeiro, Brazil}, abstract = {Stream-processing workloads and modern shared cluster environments exhibit high variability and unpredictability. Combined with the large parameter space and the diverse set of user SLOs, this makes modern streaming systems very challenging to statically configure and tune. To address these issues, in this paper we investigate a novel control-plane design, Chi, which supports continuous monitoring and feedback, and enables dynamic re-configuration. Chi leverages the key insight of embedding control-plane messages in the data-plane channels to achieve a low-latency and flexible control plane for stream-processing systems. Chi introduces a new reactive programming model and design mechanisms to asynchronously execute control policies, thus avoiding global synchronization. We show how this allows us to easily implement a wide spectrum of control policies targeting different use cases observed in production. Large-scale experiments using production workloads from a popular cloud provider demonstrate the flexibility and efficiency of our approach.}, author = {Luo Mai and Kai Zeng and Rahul Potharaju and Le Xu and Shivaram Venkataraman and Paolo Costa and Terry Kim and Saravanan Muthukrishnan and Vamsi Kuppa and Sudheer Dhulipalla and Sriram Rao} }