@inproceedings{
 title = {Integrating pig with harp to support iterative applications with fast cache and customized communication},
 type = {inproceedings},
 year = {2014},
 id = {50cba507-c12b-38fa-b8d3-1bfbf928e235},
 created = {2017-11-28T17:32:48.518Z},
 file_attached = {false},
 profile_id = {42d295c0-0737-38d6-8b43-508cab6ea85d},
 last_modified = {2020-05-11T14:43:29.410Z},
 read = {false},
 starred = {false},
 authored = {true},
 confirmed = {false},
 hidden = {false},
 citation_key = {Wu2014},
 folder_uuids = {36d8ccf4-7085-47fa-8ab9-897283d082c5},
 private_publication = {false},
 abstract = {© 2014 IEEE. Use of high-level scripting languages to solve big data problems has become a mainstream approach for sophisticated machine learning data analysis. Often data must be used in several steps of a computation to complete a full task. Composing default data transformation operators with the standard Hadoop MapReduce runtime is very convenient. However, the current strategy of using high-level languages to support iterative applications with Hadoop MapReduce relies on an external wrapper script in other languages such as Python and Groovy, which causes significant performance loss when restarting mappers and reducers between jobs. In this paper, we reduce the extra job startup overheads by integrating Apache Pig with the high-performance Hadoop plug-in Harp developed at Indiana University. This provides fast data caching and customized communication patterns among iterations for data analysis. The results show performance improvements of factors from 2 to 5.},
 bibtype = {inproceedings},
 author = {Wu, T.-L. and Koppula, A. and Qiu, J.},
 doi = {10.1109/DataCloud.2014.8},
 booktitle = {Proceedings of DataCloud 2014: 5th International Workshop on Data Intensive Computing in the Clouds - Held in Conjunction with SC 2014: The International Conference for High Performance Computing, Networking, Storage and Analysis}
}

{"_id":"K7udGgbSdA6nQfg7m","bibbaseid":"wu-koppula-qiu-integratingpigwithharptosupportiterativeapplicationswithfastcacheandcustomizedcommunication-2014","downloads":0,"creationDate":"2018-03-12T19:10:27.611Z","title":"Integrating pig with harp to support iterative applications with fast cache and customized communication","author_short":["Wu, T.","Koppula, A.","Qiu, J."],"year":2014,"bibtype":"inproceedings","biburl":"https://bibbase.org/service/mendeley/42d295c0-0737-38d6-8b43-508cab6ea85d","bibdata":{"title":"Integrating pig with harp to support iterative applications with fast cache and customized communication","type":"inproceedings","year":"2014","id":"50cba507-c12b-38fa-b8d3-1bfbf928e235","created":"2017-11-28T17:32:48.518Z","file_attached":false,"profile_id":"42d295c0-0737-38d6-8b43-508cab6ea85d","last_modified":"2020-05-11T14:43:29.410Z","read":false,"starred":false,"authored":"true","confirmed":false,"hidden":false,"citation_key":"Wu2014","folder_uuids":"36d8ccf4-7085-47fa-8ab9-897283d082c5","private_publication":false,"abstract":"© 2014 IEEE. Use of high-level scripting languages to solve big data problems has become a mainstream approach for sophisticated machine learning data analysis. Often data must be used in several steps of a computation to complete a full task. Composing default data transformation operators with the standard Hadoop MapReduce runtime is very convenient. However, the current strategy of using high-level languages to support iterative applications with Hadoop MapReduce relies on an external wrapper script in other languages such as Python and Groovy, which causes significant performance loss when restarting mappers and reducers between jobs. In this paper, we reduce the extra job startup overheads by integrating Apache Pig with the high-performance Hadoop plug-in Harp developed at Indiana University. This provides fast data caching and customized communication patterns among iterations for data analysis. The results show performance improvements of factors from 2 to 5.","bibtype":"inproceedings","author":"Wu, T.-L. and Koppula, A. and Qiu, J.","doi":"10.1109/DataCloud.2014.8","booktitle":"Proceedings of DataCloud 2014: 5th International Workshop on Data Intensive Computing in the Clouds - Held in Conjunction with SC 2014: The International Conference for High Performance Computing, Networking, Storage and Analysis","bibtex":"@inproceedings{\n title = {Integrating pig with harp to support iterative applications with fast cache and customized communication},\n type = {inproceedings},\n year = {2014},\n id = {50cba507-c12b-38fa-b8d3-1bfbf928e235},\n created = {2017-11-28T17:32:48.518Z},\n file_attached = {false},\n profile_id = {42d295c0-0737-38d6-8b43-508cab6ea85d},\n last_modified = {2020-05-11T14:43:29.410Z},\n read = {false},\n starred = {false},\n authored = {true},\n confirmed = {false},\n hidden = {false},\n citation_key = {Wu2014},\n folder_uuids = {36d8ccf4-7085-47fa-8ab9-897283d082c5},\n private_publication = {false},\n abstract = {© 2014 IEEE. Use of high-level scripting languages to solve big data problems has become a mainstream approach for sophisticated machine learning data analysis. Often data must be used in several steps of a computation to complete a full task. Composing default data transformation operators with the standard Hadoop MapReduce runtime is very convenient. However, the current strategy of using high-level languages to support iterative applications with Hadoop MapReduce relies on an external wrapper script in other languages such as Python and Groovy, which causes significant performance loss when restarting mappers and reducers between jobs. In this paper, we reduce the extra job startup overheads by integrating Apache Pig with the high-performance Hadoop plug-in Harp developed at Indiana University. This provides fast data caching and customized communication patterns among iterations for data analysis. The results show performance improvements of factors from 2 to 5.},\n bibtype = {inproceedings},\n author = {Wu, T.-L. and Koppula, A. and Qiu, J.},\n doi = {10.1109/DataCloud.2014.8},\n booktitle = {Proceedings of DataCloud 2014: 5th International Workshop on Data Intensive Computing in the Clouds - Held in Conjunction with SC 2014: The International Conference for High Performance Computing, Networking, Storage and Analysis}\n}","author_short":["Wu, T.","Koppula, A.","Qiu, J."],"biburl":"https://bibbase.org/service/mendeley/42d295c0-0737-38d6-8b43-508cab6ea85d","bibbaseid":"wu-koppula-qiu-integratingpigwithharptosupportiterativeapplicationswithfastcacheandcustomizedcommunication-2014","role":"author","urls":{},"metadata":{"authorlinks":{}},"downloads":0},"search_terms":["integrating","pig","harp","support","iterative","applications","fast","cache","customized","communication","wu","koppula","qiu"],"keywords":[],"authorIDs":[],"dataSources":["zgahneP4uAjKbudrQ","ya2CyA73rpZseyrZ8","2252seNhipfTmjEBQ"]}