{"rsdb":{"rid":"301771","subhead":"","postdate":"0","aid":"217019","fid":"57","uid":"1","topic":"1","content":"<p style=\"margin-left: 30px;\"><span style=\"font-family: 'Microsoft YaHei'; color: #808080;\"><span style=\"font-family: 'comic sans ms', sans-serif;\"><span style=\"font-size: 18px;\">\u8fd1\u671f\u63a5\u624b\u4e86\u4e0d\u5c11\u5927\u6570\u636e\u8868\u4efb\u52a1\u8c03\u5ea6\u8865\u6570\u636e\u7684\u5de5\u4f5c,\u8865\u6570\u65f6\u53d1\u73b0\u8d44\u6e90\u6d88\u8017\u5f02\u5e38\u7684\u5927\u4e14\u8fd0\u884c\u901f\u5ea6\u5374\u4e0d\u600e\u4e48\u7ed9\u529b.<\/span><\/span><\/span><\/p><br\/><p style=\"margin-left: 30px;\"><span style=\"font-family: 'Microsoft YaHei'; color: #808080;\"><span style=\"font-family: 'comic sans ms', sans-serif;\"><span style=\"font-size: 18px;\">\u53d1\u73b0\u6839\u672c\u539f\u56e0\u5728\u4e8esparkSQL\u914d\u7f6e\u6709\u8bf8\u591a\u95ee\u9898,\u89e3\u51b3\u540e\u603b\u7ed3\u51fa\u6765\u5c31\u5f53\u629b\u7816\u5f15\u7389\u4e86.<\/span><\/span><\/span><\/p><br\/><p><span style=\"font-family: 'comic sans ms', sans-serif;\"><strong>\u4ee5\u4e0b\u4e3aSparkSQL\u8c03\u4f18\u76f8\u5173\u8bbe\u7f6e<\/strong><\/span><\/p><br\/><p>\/\/1.\u4e0b\u5217Hive\u53c2\u6570\u5bf9Spark\u540c\u6837\u8d77\u4f5c\u7528\u3002<br \/>set hive.exec.dynamic.partition=true; \/\/ \u662f\u5426\u5141\u8bb8\u52a8\u6001\u751f\u6210\u5206\u533a<br \/>set hive.exec.dynamic.partition.mode=nonstrict; \/\/ \u662f\u5426\u5bb9\u5fcd\u6307\u5b9a\u5206\u533a\u5168\u90e8\u52a8\u6001\u751f\u6210<br \/>set hive.exec.max.dynamic.partitions = 100; \/\/ \u52a8\u6001\u751f\u6210\u7684\u6700\u591a\u5206\u533a\u6570<\/p><br\/><p>\/\/3.executor\u80fd\u529b<br \/>set spark.executor.memory; \/\/ executor\u7528\u4e8e\u7f13\u5b58\u6570\u636e\u3001\u4ee3\u7801\u6267\u884c\u7684\u5806\u5185\u5b58\u4ee5\u53caJVM\u8fd0\u884c\u65f6\u9700\u8981\u7684\u5185\u5b58<br \/>set spark.yarn.executor.memoryOverhead; \/\/Spark\u8fd0\u884c\u8fd8\u9700\u8981\u4e00\u4e9b\u5806\u5916\u5185\u5b58\uff0c\u76f4\u63a5\u5411\u7cfb\u7edf\u7533\u8bf7\uff0c\u5982\u6570\u636e\u4f20\u8f93\u65f6\u7684netty\u7b49\u3002<br \/>set spark.sql.windowExec.buffer.spill.threshold; \/\/\u5f53\u7528\u6237\u7684SQL\u4e2d\u5305\u542b\u7a97\u53e3\u51fd\u6570\u65f6\uff0c\u5e76\u4e0d\u4f1a\u628a\u4e00\u4e2a\u7a97\u53e3\u4e2d\u7684\u6240\u6709\u6570\u636e\u5168\u90e8\u8bfb\u8fdb\u5185\u5b58\uff0c\u800c\u662f\u7ef4\u62a4\u4e00\u4e2a\u7f13\u5b58\u6c60\uff0c\u5f53\u6c60\u4e2d\u7684\u6570\u636e\u6761\u6570\u5927\u4e8e\u8be5\u53c2\u6570\u8868\u793a\u7684\u9608\u503c\u65f6\uff0cspark\u5c06\u6570\u636e\u5199\u5230\u78c1\u76d8<br \/>set spark.executor.cores; \/\/\u5355\u4e2aexecutor\u4e0a\u53ef\u4ee5\u540c\u65f6\u8fd0\u884c\u7684task\u6570<\/p><br\/>","orderid":"0","title":"SparkSQL\u6267\u884c\u65f6\u53c2\u6570\u4f18\u5316","smalltitle":"","mid":"0","fname":"\u6570\u636e\u5e93\u7f16\u7a0b","special_id":"0","bak_id":"0","info":"0","hits":"177","pages":"1","comments":"0","posttime":"2019-04-02 22:08:36","list":"1554214116","username":"admin","author":"","copyfrom":"","copyfromurl":"","titlecolor":"","fonttype":"0","titleicon":"0","picurl":"https:\/\/www.cppentry.com\/upload_files\/","ispic":"0","yz":"1","yzer":"","yztime":"0","levels":"0","levelstime":"0","keywords":"<A HREF='https:\/\/www.cppentry.com\/do\/search.php?type=keyword&keyword=SparkSQL' target=_blank>SparkSQL<\/A> <A HREF='https:\/\/www.cppentry.com\/do\/search.php?type=keyword&keyword=%D0%D0%CA%B1' target=_blank>\u884c\u65f6<\/A> <A HREF='https:\/\/www.cppentry.com\/do\/search.php?type=keyword&keyword=%B2%CE%CA%FD' target=_blank>\u53c2\u6570<\/A> <A HREF='https:\/\/www.cppentry.com\/do\/search.php?type=keyword&keyword=%D3%C5%BB%AF' target=_blank>\u4f18\u5316<\/A>","jumpurl":"","iframeurl":"","style":"","template":"a:3:{s:4:\"head\";s:0:\"\";s:4:\"foot\";s:0:\"\";s:8:\"bencandy\";s:0:\"\";}","target":"0","ip":"47.106.78.186","lastfid":"0","money":"0","buyuser":"","passwd":"","allowdown":"","allowview":"","editer":"","edittime":"0","begintime":"0","endtime":"0","description":"SparkSQL\u6267\u884c\u65f6\u53c2\u6570\u4f18\u5316","lastview":"1713520296","digg_num":"0","digg_time":"0","forbidcomment":"0","ifvote":"0","heart":"","htmlname":"","city_id":"0"},"page":"1"}