@conference {82, title = {Autoscaling for Hadoop Clusters}, year = {2016}, abstract = {

Unforeseen events such as node failures and re- source contention can have a severe impact on the performance of data processing frameworks, such as Hadoop, especially in cloud environments where such incidents are common. SLA compliance in the presence of such events requires the ability to quickly and dynamically resize infrastructure resources. Unfortunately, the distributed and stateful nature of data processing frameworks makes it challenging to accurately scale the system at run-time. In this paper, we present the design and implementation of a model-driven autoscaling solution for Hadoop clusters. We first develop novel gray-box performance models for Hadoop workloads that specifically relate job execution times to resource allocation and workload parameters. We then employ these models to dynamically determine the resources required to successfully complete the Hadoop jobs as per the user-specified SLA under various scenarios including node failures and multi-job executions. Our experimental results on three different Hadoop cloud clusters and across different workloads demonstrate the efficacy of our models and highlight their autoscaling capabilities.

}, author = {Anshul Gandhi and Sidhartha Thota and Parijat Dube and Andrzej Kochut and Li Zhang} } @conference {81, title = {Using Machine Learning for Black-Box Autoscaling}, year = {2016}, abstract = {

Autoscaling is the practice of automatically adding or removing resources for an application deployment to meet performance targets in response to changing workload conditions. However, existing autoscaling approaches typically require expert application and system knowledge to minimize resource costs and performance target violations, thus limiting their applicability. We present MLscale, an application-agnostic, machine learning based autoscaler that is composed of: (i) a neural network based online (black-box) performance modeler, and (ii) a regression based metrics predictor to estimate post-scaling application and system metrics. Implementation results for diverse applications across several traces highlight MLscale{\textquoteright}s application-agnostic behavior and show that MLscale (i) reduces resource costs by about 50\% compared to the optimal static policy, (ii) is within 15\% of the cost of the optimal dynamic policy, and (iii) provides similar cost-performance tradeoffs, without requiring any tuning, when compared to carefully tuned threshold-based policies.

}, author = {Muhammad Wajahat and Anshul Gandhi and Alexei Karve and Andrzej Kochut} }