{"rsdb":{"rid":"309149","subhead":"","postdate":"0","aid":"224311","fid":"120","uid":"1","topic":"1","content":"<div id=\"article_content\" class=\"article_content clearfix csdn-tracking-statistics\" data-pid=\"blog\" data-mod=\"popu_307\" data-dsm=\"post\">\n            <link rel=\"stylesheet\" href=\"https:\/\/csdnimg.cn\/release\/phoenix\/template\/css\/ck_htmledit_views-f57960eb32.css\">\n                              <link rel=\"stylesheet\" href=\"https:\/\/csdnimg.cn\/release\/phoenix\/template\/css\/ck_htmledit_views-f57960eb32.css\">\n          <div class=\"htmledit_views\" id=\"content_views\">\n            <div class=\"content-detail markdown-body\">\n                    \n<h2>1. \u95ee\u9898\u80cc\u666f<\/h2>\n<p>\u6211\u4eec\u5728\u4f7f\u7528Kafka\u7684\u65f6\u5019\uff0c\u6709\u65f6\u5019\u4f1a\u9047\u5230\u4ee5\u4e0b\u51e0\u79cd\u573a\u666f\uff1a<\/p>\n<ul><li>\u539f\u6709Kafka\u96c6\u7fa4\u673a\u578b\u914d\u7f6e\u8fc7\u65e7\uff0c\u9700\u8981\u5347\u7ea7\u6362\u4ee3\uff0c\u91cd\u65b0\u89c4\u5212\u4e00\u4e2aKafka\u96c6\u7fa4\uff0c\u5c06\u8001\u96c6\u7fa4\u7684\u6570\u636e\u8fc1\u79fb\u5230\u65b0\u96c6\u7fa4\u4e0a<\/li>\n<li>\u6570\u636e\u4e0a\u4e91\uff0c\u4e91\u4e0bKafka\u96c6\u7fa4\u6570\u636e\u8fc1\u79fb\u5230\u4e91\u4e0aKafka\u96c6\u7fa4\/Kafka\u670d\u52a1<\/li>\n<li>\u591a\u4e2aKafka\u96c6\u7fa4\u6570\u636e\u6c47\u603b\u5230\u4e00\u4e2aKafka\u96c6\u7fa4<\/li>\n<li>\u57fa\u4e8e\u4e1a\u52a1Kafka\u96c6\u7fa4\uff0c\u6784\u5efa\u4e00\u4e2aKafka\u707e\u5907\u96c6\u7fa4<\/li>\n<\/ul><p>\u603b\u7ed3\u4e00\u4e0b\uff0c\u4ee5\u4e0a\u573a\u666f\u53ef\u4ee5\u62bd\u8c61\u6210\u4e24\u7c7b\uff1a<\/p>\n<ul><li>\u6570\u636e\u8fc1\u79fb<\/li>\n<li>\u6570\u636e\u707e\u5907<\/li>\n<\/ul><p>Kafka\u793e\u533a\u63d0\u4f9b\u4e86\u4e00\u4e2a\u5de5\u5177\uff0c\u5373MirrorMaker\uff0c\u5b83\u53ef\u4ee5\u6ee1\u8db3\u7528\u6237\u7684\u6570\u636e\u8fc1\u79fb\u9700\u6c42\uff0c\u540c\u65f6\u4e00\u5b9a\u7a0b\u5ea6\u7684\u6ee1\u8db3\u6570\u636e\u707e\u5907\u9700\u6c42\u3002\u5f53\u7136\u9664\u4e86\u539f\u751fMirrorMaker\u5de5\u5177\uff0c\u4e5f\u5b58\u5728\u7740\u5404\u79cd\u884d\u751f\u7248\u672c\u7684\u6570\u636e\u540c\u6b65\u5de5\u5177\u3002\u4e0b\u9762\u5c31\u7b80\u5355\u4ecb\u7ecd\u4e00\u4e0b\u793e\u533a\u7248\u7684MirrorMaker\u5de5\u5177\u3002<\/p>\n<h2>2. MirrorMaker\u5de5\u5177\u4ecb\u7ecd<\/h2>\n<p><img src=\"https:\/\/yqfile.alicdn.com\/c14665439b67390fc36fd603e7e5cf7e5f0f4a33.png\" alt=\"image\" title=\"image\"><\/p>\n<p>MirrorMaker\u5de5\u5177\u4e0d\u8fc7\u662f\u5c06Kafka Consumer\u548cProducer\u6253\u5305\u5728\u4e00\u8d77\u3002\u4e00\u4e2aMirrorMaker\u8fdb\u7a0b\u4f1a\u8d77\u82e5\u5e72\u4e2aConsumer\u548c\u4e00\u4e2aProducer\uff0cConsumer\u8d1f\u8d23\u4ece\u6e90Kafka\u96c6\u7fa4\u6d88\u8d39\u6570\u636e\uff0cProducer\u8d1f\u8d23\u5c06\u6570\u636e\u5199\u5230\u76ee\u6807Kafka\u96c6\u7fa4\u3002<\/p>\n<blockquote><p>The source and destination clusters are completely independent entities: they can have different numbers of partitions and the offsets will not be the same.<\/p><\/blockquote>\n<p>\u4e0a\u9762\u662fKafka\u5b98\u65b9\u6587\u6863\u539f\u6587\u3002\u5728\u76ee\u6807Kafka\u96c6\u7fa4\u4e2d\uff0cTopic\u53ef\u4ee5\u6709\u4e0d\u540c\u7684\u5206\u533a\u6570\uff0coffset\u4e5f\u53ef\u80fd\u4e0d\u4e00\u6837\u3002<\/p>\n<h2>3. \u5982\u4f55\u521b\u5efa\u955c\u50cf\u96c6\u7fa4<\/h2>\n<p>\u5728E-MapReduce\u4e0a\u521b\u5efa\u4e24\u4e2aKafka\u96c6\u7fa4\uff0c\u8fd9\u91cc\u4f7f\u7528EMR-3.18.1\u7248\u672c\uff0c\u5206\u522b\u547d\u540d\u4e3aKafka-A\u548cKafka-B\u3002\u5efa\u8bae\u5c06\u8fd9\u4e24\u4e2a\u96c6\u7fa4\u5efa\u5728\u4e00\u4e2a\u5b89\u5168\u7ec4\u5185\uff0c\u907f\u514d\u7f51\u7edc\u914d\u7f6e\u95ee\u9898\u3002<\/p>\n<p>\u5728Kafka-A \u96c6\u7fa4\u521b\u5efa\u4e00\u4e2aTopic \"test\"\uff1a<\/p>\n<pre><code>kafka-topics.sh --zookeeper emr-header-1:2181\/kafka-1.1.1 --topic test --partitions 100 \n--replication-factor 2 --create<\/code><\/pre>\n<p>\u5e76\u5411\u5176\u4e2d\u53d1\u9001\u6570\u636e\uff1a<\/p>\n<pre><code>kafka-producer-perf-test.sh --topic test --num-records 10000000000 --throughput 10000 \n--producer-props bootstrap.servers=emr-worker-1:9092 --record-size 10<\/code><\/pre>\n<p>\u5728Kafka-B\u96c6\u7fa4\u540c\u6837\u521b\u5efa\u4e00\u4e2aTopic \"test\"\uff0c\u6539\u53d8\u4e00\u4e0b\u5206\u533a\u6570\u76ee\uff1a<\/p>\n<pre><code>kafka-topics.sh --zookeeper emr-header-1:2181\/kafka-1.1.1 --topic test --partitions 50 \n--replication-factor 2 --create<\/code><\/pre>\n<p>\u540c\u65f6\u5728Kafka-B\u96c6\u7fa4\u542f\u52a8\u4e00\u4e2aMirrorMaker\u8fdb\u7a0b\uff0c\u8fd9\u4e2a\u8fdb\u7a0b\u4f1a\u542f\u52a84\u4e2a\u6d88\u8d39\u7ebf\u7a0b\u6d88\u8d39\u6570\u636e\uff1a<\/p>\n<pre><code>kafka-mirror-maker.sh --consumer.config consumer.conf --producer.config producer.conf \n--new.consumer --num.streams 4 --whitelist test<\/code><\/pre>\n<p>\u8fd9\u65f6\uff0c\u6211\u4eec\u5c31\u53ef\u4ee5\u5728Kafka-B\u96c6\u7fa4\u770b\u5230\u540c\u6b65\u8fc7\u6765\u7684\u6570\u636e\u3002\u6211\u4eec\u5728Kafka-A\u53d1\u9001\u4e00\u6761\u6709\u610f\u4e49\u7684\u6570\u636e\u201chello world\uff01\u201d\uff0c\u5728Kafka-B\u6d88\u8d39\u770b\u770b\uff1a<br><img src=\"https:\/\/yqfile.alicdn.com\/1f33534431ce7d1ff053badcc0eaf93f46d43a5e.png\" alt=\"image\" title=\"image\"><\/p>\n<p><strong>\u6ce8\u610f\uff1a<\/strong><\/p>\n<ul><li>\u5728\u5b9e\u9645\u4f7f\u7528\u4e2d\uff0c\u5efa\u8bae\u4e0d\u8981\u5f00\u542f\"auto.topic.create\"\uff0c\u867d\u7136\u8fd9\u6837\u907f\u514d\u6211\u4eec\u5728\u76ee\u6807\u96c6\u7fa4\u624b\u52a8\u521b\u5efatopic\uff0c\u5e76\u4e14\u53ef\u4ee5\u81ea\u52a8\u5e94\u5bf9\u65b0\u589etopic\u3002\u4f46\u662f\u9ed8\u8ba4\u521b\u5efa\u7684topic\uff0c\u5206\u533a\u6570\u76ee\uff0c\u526f\u672c\u6570\u90fd\u662f\u56fa\u5b9a\u7684\uff0c\u4e0d\u4e00\u5b9a\u5c31\u5408\u9002\u3002\u5efa\u8bae\u6309\u7167\u5b9e\u9645\u9700\u6c42\u624b\u52a8\u521b\u5efatopic\uff0c\u6216\u8005\u81f3\u5c11\u624b\u52a8\u521b\u5efa\u90a3\u4e9b\u6838\u5fc3topic\uff0c\u81ea\u52a8\u521b\u5efa\u90a3\u4e9b\u975e\u6838\u5fc3topic\u3002<\/li>\n<li>\u4e00\u4e2aMirrorMaker\u8fdb\u7a0b\u7684\u5904\u7406\u80fd\u529b\u662f\u6709\u9650\u7684\uff0c\u6211\u4eec\u53ef\u4ee5\u5728\u4e00\u53f0\u673a\u5668\u591a\u542f\u52a8\u51e0\u4e2aMirrorMaker\u8fdb\u7a0b\u3002<\/li>\n<li>MirrorMaker\u6240\u5728\u7684\u673a\u5668\u7684\u5e26\u5bbd\u80fd\u529b\u662f\u6709\u9650\u7684\uff0c\u6211\u4eec\u53ef\u4ee5\u5728\u591a\u53f0\u673a\u5668\u540c\u65f6\u90e8\u7f72MirrorMaker\u8fdb\u7a0b\u3002<\/li>\n<li>\u6240\u6709\u7684MirrorMaker\u8fdb\u7a0b\u5fc5\u9009\u4f7f\u7528\u76f8\u540c\u7684\u914d\u7f6e\u3002<\/li>\n<\/ul><h2>4. MirrorMaker\u53c2\u6570\u8bf4\u660e<\/h2>\n<p>\u7b80\u5355\u4ecb\u7ecd\u4e0b\u5de5\u5177\u7684\u5404\u4e2a\u53c2\u6570\uff1a<\/p>\n<table><thead><tr><th>\u914d\u7f6e\u9879<\/th>\n<th>\u8bf4\u660e<\/th>\n<\/tr><\/thead><tbody><tr><td>abort.on.send.failure<\/td>\n<td>\u5f53\u53d1\u9001\u6570\u636e\u5931\u8d25\u65f6\uff0cMirrorMaker\u8fdb\u7a0b\u9000\u51fa\uff0c\u9ed8\u8ba4true\u3002<\/td>\n<\/tr><tr><td>blacklist<\/td>\n<td>\u65e0\u9700\u590d\u5236\u6570\u636e\u7684Topic\u5217\u8868\uff0c\u652f\u6301Java\u6b63\u5219\u8868\u8fbe\u3002\u6ce8\u610f\uff0c\u53ea\u6709\u4f7f\u7528\u65e7\u7684Consumer\u65f6\u624d\u751f\u6548\u3002<\/td>\n<\/tr><tr><td>whitelist<\/td>\n<td>\u9700\u8981\u590d\u5236\u6570\u636e\u7684Topic\u5217\u8868\uff0c\u652f\u6301Java\u6b63\u5219\u8868\u8fbe\u3002<\/td>\n<\/tr><tr><td>consumer.config<\/td>\n<td>Consumer\u5ba2\u6237\u7aef\u914d\u7f6e\u6587\u4ef6\u3002\u6ce8\u610f\uff0c\u5f53\u914d\u7f6e\u4e86\"zookeeper.connect\"\u65f6\u8868\u793a\u4f7f\u7528\u65e7\u7684Consumer\u3002<\/td>\n<\/tr><tr><td>producer.config<\/td>\n<td>Producer\u5ba2\u6237\u7aef\u914d\u7f6e\u6587\u4ef6<\/td>\n<\/tr><tr><td>consumer.rebalance.listener<\/td>\n<td>\u81ea\u5b9a\u4e49\u7684consumer rebalance listener\uff0c\u4e00\u822c\u65e0\u9700\u914d\u7f6e<\/td>\n<\/tr><tr><td>rebalance.listener.args<\/td>\n<td>\u81ea\u5b9a\u4e49\u7684consumer rebalance listener\u53c2\u6570<\/td>\n<\/tr><tr><td>message.handler<\/td>\n<td>\u81ea\u5b9a\u4e49\u7684\u6d88\u606f\u5904\u7406\u5668\uff0c\u6e90\u7aef\u6570\u636e\u7ecf\u8fc7\u5904\u7406\u540e\u518d\u5199\u5230\u76ee\u7684\u7aef<\/td>\n<\/tr><tr><td>message.handler.args<\/td>\n<td>\u81ea\u5b9a\u4e49\u7684\u6d88\u606f\u5904\u7406\u5668\u53c2\u6570<\/td>\n<\/tr><tr><td>new.consumer<\/td>\n<td>\u6307\u5b9a\u4f7f\u7528\u65b0\u7684Consumer<\/td>\n<\/tr><tr><td>num.streams<\/td>\n<td>Consumer\u5e76\u53d1\u6570<\/td>\n<\/tr><tr><td>offset.commit.interval.ms<\/td>\n<td>Offset commit\u95f4\u9694<\/td>\n<\/tr><\/tbody><\/table><p><strong>\u6ce8\u610f\uff1a<\/strong><\/p>\n<ul><li>\u76ee\u6807Kafka\u96c6\u7fa4\u9700\u8981\u540c\u6b65\u6e90Kakfa\u96c6\u7fa4\u7684\u914d\u7f6e\uff0c\u8b6c\u5982\u6700\u5927\u6d88\u606fSize\u7b49\u3002<\/li>\n<li>\n<p>Consumer\u914d\u7f6e\uff1a<\/p>\n<ul><li>\u6839\u636e\u6700\u5927\u6d88\u606f\uff0c\u8c03\u6574max.partition.fetch.bytes\u548cfetch.max.bytes\u53c2\u6570\u7b49\u7b49\u3002<\/li>\n<li>\u6839\u636e\u7f51\u7edc\u73af\u5883\uff0c\u9002\u5f53\u589e\u5927session.timeout.ms\u548crequest.timeout.ms\u3002\u5f53\u7f51\u7edc\u73af\u5883\u6076\u52a3\u65f6\uff0c\u9002\u5f53\u589e\u52a0heartbeat.interval.ms\uff0c\u907f\u514d\u4e0d\u5fc5\u8981\u7684rebalance\uff0c\u5f53\u7f51\u7edc\u73af\u5883\u6bd4\u8f83\u597d\u65f6\uff0c\u5219\u9002\u5f53\u51cf\u5c0fheartbeat.interval.ms\uff0c\u4ee5\u4f7f\u5f97\u670d\u52a1\u7aef\u66f4\u5feb\u54cd\u5e94Client\u7684\u5f02\u5e38\u3002<\/li>\n<\/ul><\/li>\n<li>\n<p>Producer\u914d\u7f6e\uff1a<\/p>\n<ul><li>\u6839\u636e\u6700\u5927\u6d88\u606f\uff0c\u8c03\u6574max.request.size\u7b49\u53c2\u6570\u3002<\/li>\n<li>reties=Int.MAX_VALUE\uff0cacks=-1(all)\u3002<\/li>\n<li>\u6839\u636e\u7f51\u7edc\u73af\u5883\uff0c\u9002\u5f53\u589e\u5927request.timeout.ms\u3002<\/li>\n<\/ul><\/li>\n<\/ul><h2>5. \u5c0f\u7ed3<\/h2>\n<p>\u672c\u6587\u4ecb\u7ecd\u4e86\u5982\u4f55\u4f7f\u7528\u793e\u533a\u7684Kafka MirrorMaker\u5de5\u5177\u8fdb\u884c\u96c6\u7fa4\u95f4\u7684\u6570\u636e\u590d\u5236\u3002E-MapReduce Kafka\u57fa\u4e8eKafka Conenct\u5b9e\u73b0\u4e86\u65b0\u7684\u4e00\u79cd\u6570\u636e\u540c\u6b65\u5de5\u5177\uff0c\u652f\u6301\u6570\u636e\u590d\u5236\u548c\u707e\u5907\u96c6\u7fa4\u6784\u5efa\u3002Kafka\u793e\u533a\u5f53\u524d\u6b63\u5728\u8fdb\u884cMirrorMaker v2.0\u7248\u672c\u5f00\u53d1\uff0c\u540c\u6837\u4e5f\u662f\u57fa\u4e8eKafka Conenct\u6765\u5b9e\u65bd\u3002\u540e\u7eed\u5c06\u4e13\u95e8\u5199\u4e00\u7bc7\u6587\u7ae0\u6765\u4ecb\u7ecd\u3002<\/p>\n\n        \n        \n                \n    <\/div>          <\/div>\n                  <\/div>","orderid":"0","title":"\n &nbsp;  &nbsp;  &nbsp;  &nbsp;  &nbsp;  &nbsp;  &nbsp;  &nbsp;  &nbsp;  &nbsp;   \u5982\u4f55\u5728E-MapReduce\u4e0a\u8fdb\u884cKafka\u96c6\u7fa4\u95f4\u6570\u636e\u590d\u5236\n &nbsp;  ","smalltitle":"","mid":"0","fname":"Kafka","special_id":"0","bak_id":"0","info":"0","hits":"798","pages":"1","comments":"0","posttime":"2019-05-16 02:16:28","list":"1557944188","username":"admin","author":"","copyfrom":"","copyfromurl":"","titlecolor":"","fonttype":"0","titleicon":"0","picurl":"https:\/\/www.cppentry.com\/upload_files\/","ispic":"0","yz":"1","yzer":"","yztime":"0","levels":"0","levelstime":"0","keywords":"<A HREF='https:\/\/www.cppentry.com\/do\/search.php?type=keyword&keyword=%26nbsp' target=_blank>&nbsp<\/A> <A HREF='https:\/\/www.cppentry.com\/do\/search.php?type=keyword&keyword=%BA%CE%D4%DA' target=_blank>\u4f55\u5728<\/A> <A HREF='https:\/\/www.cppentry.com\/do\/search.php?type=keyword&keyword=E-MapReduce' target=_blank>E-MapReduce<\/A> <A HREF='https:\/\/www.cppentry.com\/do\/search.php?type=keyword&keyword=%BD%F8%D0%D0' target=_blank>\u8fdb\u884c<\/A> <A HREF='https:\/\/www.cppentry.com\/do\/search.php?type=keyword&keyword=Kafka' target=_blank>Kafka<\/A> <A HREF='https:\/\/www.cppentry.com\/do\/search.php?type=keyword&keyword=%BC%AF%C8%BA' target=_blank>\u96c6\u7fa4<\/A> <A HREF='https:\/\/www.cppentry.com\/do\/search.php?type=keyword&keyword=%CA%FD%BE%DD' target=_blank>\u6570\u636e<\/A> <A HREF='https:\/\/www.cppentry.com\/do\/search.php?type=keyword&keyword=%B8%B4%D6%C6' target=_blank>\u590d\u5236<\/A>","jumpurl":"","iframeurl":"","style":"","template":"a:3:{s:4:\"foot\";s:0:\"\";s:8:\"bencandy\";s:0:\"\";s:4:\"head\";s:0:\"\";}","target":"0","ip":"47.106.78.186","lastfid":"0","money":"0","buyuser":"","passwd":"","allowdown":"","allowview":"","editer":"","edittime":"0","begintime":"0","endtime":"0","description":"                                                                                    1. \u95ee\u9898\u80cc\u666f\u6211\u4eec\u5728\u4f7f\u7528Kafka\u7684\u65f6\u5019\uff0c\u6709\u65f6\u5019\u4f1a\u9047\u5230\u4ee5\u4e0b\u51e0\u79cd\u573a\u666f\uff1a\u539f\u6709Kafka\u96c6\u7fa4\u673a\u578b\u914d\u7f6e\u8fc7\u65e7\uff0c\u9700\u8981\u5347\u7ea7\u6362\u4ee3\uff0c\u91cd\u65b0\u89c4\u5212\u4e00\u4e2aKafka\u96c6\u7fa4\uff0c\u5c06\u8001\u96c6\u7fa4\u7684\u6570\u636e\u8fc1\u79fb\u5230\u65b0\u96c6\u7fa4\u4e0a\u6570\u636e\u4e0a\u4e91\uff0c..","lastview":"1714157473","digg_num":"494","digg_time":"1714132999","forbidcomment":"0","ifvote":"0","heart":"","htmlname":"","city_id":"0"},"page":"1"}