{"rsdb":{"rid":"309146","subhead":"","postdate":"0","aid":"224308","fid":"119","uid":"1","topic":"1","content":"
\n
\n \n \u7248\u6743\u58f0\u660e\uff1a\u672c\u6587\u4e3a\u535a\u4e3b\u539f\u521b\u6587\u7ae0\uff0c\u672a\u7ecf\u535a\u4e3b\u5141\u8bb8\u4e0d\u5f97\u8f6c\u8f7d\u3002 https:\/\/blog.csdn.net\/github_37643896\/article\/details\/87258584 <\/div>\n \n \n
\n

<\/p>\n\n

\u4e2a\u4eba\u4f7f\u7528flume \u76f8\u5bf9\u8f83\u591a \uff0c\u5bf9\u4ed6\u7684\u91c7\u96c6\u4efb\u52a1\u6bd4\u8f83\u559c\u6b22 ,\u81ea\u5df1\u505a\u4e86\u4e00\u4e9b\u6570\u636e\u5e93\u65b9\u5411\u7684\u7684\u62d3\u5c55\u3002<\/p>\n\n

\u867d\u7136 github \u4e0a \u6709\u5f88\u591a \u81ea\u5b9a\u4e49\u7684 flume sql-source \u6bd4\u5982 \u5927\u540d\u9f0e\u9f0e \u7684https:\/\/github.com\/keedio\/flume-ng-sql-source<\/a><\/p>\n\n

\u4f46\u662f \u6211\u4e2a\u4eba\u5728\u4f7f\u7528\u7684\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u4e00\u4e9b\u95ee\u9898 \u4e5f\u4e00\u76f4\u6ca1\u6709\u5f97\u5230\u89e3\u51b3 \uff0chttps:\/\/github.com\/keedio\/flume-ng-sql-source\/issues\/59<\/a><\/p>\n\n

\u5e76\u4e14 \u683c\u5f0f \u662fcsv \u683c\u5f0f \u3002<\/p>\n\n

\u6240\u4ee5\u6211\u81ea\u5df1\u5199\u4e86\u4e00\u5957 sqlsource \u57fa\u4e8e\u539f\u751f\u7684jdbc \u8fde\u63a5 \uff0c\u6ca1\u6709\u4f7f\u7528 \u5176\u4ed6\u7684\u6846\u67b6 \uff0c\u8bfb\u53d6\u6570\u636e\u5e93 \u8f6c\u4e3a json \u683c\u5f0f \u3002\u4e5f\u4fbf\u4e8e\u89e3\u6790\u4e2a\u4eba \u9879\u76ee \u5730\u5740<\/a>\uff0c\u6b22\u8fce\u63d0\u51fa\u4e0d\u8db3 \uff0c\u6709\u95ee\u9898 \u76f4\u63a5 \u63d0issues\u3002\u6211\u4f1a\u5c3d\u5feb\u5e2e\u5927\u5bb6\u89e3\u51b3 \u3002<\/p>\n\n

\u4e00\u3001
\n\u5728\u4f7f\u7528flume\u91c7\u96c6\u65e5\u5fd7\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7flume\u8fdb\u884c\u76d1\u63a7\u67d0\u4e00\u4e2a\u6587\u4ef6\u628a\u751f\u4ea7\u7684\u6570\u636e\u4f20\u8f93\u7ed9\u6307\u5b9a\u7684sink\uff0c\u4f46\u662f\u5982\u679c\u67d0\u6bb5\u65f6\u95f4flume\u6240\u5728\u673a\u5668\u5b95\u673a\u4e86\uff0c\u90a3\u4e48\u5f53\u91cd\u65b0\u542f\u52a8\u540e\uff0c\u5728\u53bb\u76d1\u63a7\u65f6\uff0c\u4f1a\u5bfc\u81f4\u6709\u6570\u636e\u4e22\u5931\uff0c\u4e0d\u662f\u63a5\u7740\u4e0a\u4e00\u6b21\u7684\u6570\u636e\u7ee7\u7eed\u8fdb\u884c\u8bfb\u53d6\uff0c\u56e0\u6b64\u9488\u5bf9\u8fd9\u79cd\u60c5\u51b5\u65f6\u53ef\u80fd\u9700\u8981\u6211\u4eec\u81ea\u5b9a\u4e49\u4e00\u4e2asource\uff0c\u8bb0\u5f55\u504f\u79fb\u91cf\uff0c\u6bcf\u6b21\u90fd\u662f\u63a5\u7740\u4e0a\u6b21\u7ee7\u7eed\u8bfb\uff0c\u8bb0\u5f55 \u6570\u636e\u5df2\u7ecf\u53d1\u9001\u7684\u4f4d\u7f6e \u3002
\n\u4e8c\u3001
\n\u4e0b\u9762\u5c31\u662f\u5177\u4f53\u5b9e\u73b0\u7684\u4ee3\u7801
\n\u518d\u5199\u4ee3\u7801\u65f6\u53ef\u4ee5\u53c2\u7167\u5b98\u65b9\u7ed9\u7684source\u7684\u6e90\u7801\u8fdb\u884c\u7f16\u5199.
\nflume\u7684\u751f\u547d\u5468\u671f\uff1a \u5148\u6267\u884c\u6784\u9020\u5668\uff0c\u518d\u6267\u884c config\u65b9\u6cd5 --> start\u65b9\u6cd5 --> processor.process\u2013> stop<\/p>\n\n

\u8bfb\u53d6\u914d\u7f6e\u6587\u4ef6 -> \u521d\u59cb\u5316 \u6570\u636e\u5e93\u8fde\u63a5\u53ca \u76f8\u5173\u53c2\u6570 -> \u89e3\u6790 sql -> \u83b7\u53d6 resultsSet -> \u8f6c\u4e3a Json -> \u53d1\u9001\u4e2achannel -> stop -> \u8bb0\u5f55\u5f53\u524d\u4f4d\u7f6e<\/p>\n\n

\u5177\u4f53 \u8be6\u89c1https:\/\/github.com\/HbnKing\/Flume-ng-Database\/blob\/master\/RDB\/src\/main\/java<\/a>\/com\/hbn\/rdb\/source\/SQLSource.java<\/a><\/a><\/p>\n\n

\u4e09 \u3001<\/p>\n\n

\u914d\u7f6e\u6587\u4ef6<\/p>\n\n

\u8fd9\u91cc\u91c7\u7528 loggersink \u4e3b\u8981\u770b\u4e00\u4e0bSource \u7684 \u4e00\u4e9b\u76f8\u5173 \u914d\u7f6e<\/p>\n\n

\u652f\u6301 \u81ea\u5b9a\u4e49 sql <\/p>\n\n

\nf2.sources = r1\nf2.channels = c1\nf2.sinks = k1\n\n# \u8fd9\u91cc\u7528 \u81ea\u5df1\u5b9a\u4e49\u7684 SQLSource\nf2.sources.r1.type = com.hbn.rdb.source.SQLSource\nf2.sources.r1.connectionurl = jdbc:oracle:thin:@\/\/ip:1521\/orcl\nf2.sources.r1.user = yyj\nf2.sources.r1.password = yyj\nf2.sources.r1.driverclass= oracle.jdbc.driver.OracleDriver\nf2.sources.r1.filepath = \/var\/log\/sqllog\nf2.sources.r1.filename = sqlSource.status\n#f2.sources.r1.customquery = select a.id,a.COUPON_id,b.id from USER_COUPON_CODE_1 a ,COUPON_CODE b  where a.COUPON_id = b.id\n#f2.sources.r1.customquery = select a.id,a.COUPON_id,b.id from USER_COUPON_CODE_1 a ,COUPON_CODE b  where a.COUPON_id = b.id  and  a.id > $@$\nf2.sources.r1.customquery = select * from USER_COUPON_CODE_1\nf2.sources.r1.begin = 0\nf2.sources.r1.autoincrementfield = a.id\nf2.sources.r1.batchsize = 1000\n#\u5177\u4f53\u5b9a\u4e49channel\nf2.channels.c1.type = memory\nf2.channels.c1.capacity = 1000\nf2.channels.c1.transactionCapacity = 100\n#\u5177\u4f53\u5b9a\u4e49sink\nf2.sinks.k1.type = logger\n#\u7ec4\u88c5source\u3001channel\u3001sink\nf2.sources.r1.channels = c1\nf2.sinks.k1.channel = c1<\/code><\/pre>\n\n
\u56db \u3001<\/p>\n\n
\u542f\u52a8\u547d\u4ee4<\/p>\n\n
bin\/flume-ng agent --conf conf\/ --conf-file conf\/flume-conf1.properties --name f2-Dflume.root.logger=INFO,console<\/p>\n\n
<\/p>\n\n
\u4e94 \u3001<\/p>\n\n
\u67e5\u770b\u7ed3\u679c<\/p>\n\n
\n\n{  \"ID\" : NumberLong(\"10000900000\"), \"COUPON_ID\" : 900000, \"ID_3\" : NumberLong(\"10000900000\") }\n{ \"ID\" : NumberLong(\"10000700000\"), \"COUPON_ID\" : 700000, \"ID_3\" : NumberLong(\"10000700000\") }\n{  \"ID\" : NumberLong(\"10000300000\"), \"COUPON_ID\" : 300000, \"ID_3\" : NumberLong(\"10000300000\") }\n{ \"ID\" : NumberLong(\"10000800000\"), \"COUPON_ID\" : 800000, \"ID_3\" : NumberLong(\"10000800000\") }\n{ \"ID\" : NumberLong(\"10000500001\"), \"COUPON_ID\" : 500001, \"ID_3\" : NumberLong(\"10000500001\") }\n{  \"ID\" : NumberLong(\"10000500002\"), \"COUPON_ID\" : 500002, \"ID_3\" : NumberLong(\"10000500002\") }\n{  \"ID\" : NumberLong(\"10000500003\"), \"COUPON_ID\" : 500003, \"ID_3\" : NumberLong(\"10000500003\") }<\/code><\/pre>\n\n
\u516d \u3001<\/p>\n\n
\u5176\u4ed6 <\/p>\n\n
\u5982\u6709\u95ee\u9898 \uff0c\u8bf7\u7559\u8a00 \uff0c\u6b22\u8fce\u63d0\u51fa\u4e0d\u8db3 \u548c \u5efa\u8bae \u3002<\/p>\n\n
\u611f\u8c22\u8d21\u732e \u3002<\/p>\n          <\/div>\n                  <\/div>","orderid":"0","title":"Flume  \u81ea\u5b9a\u4e49source   -- SQLSource  (\u8f6c\u4e3a json \u683c\u5f0f)","smalltitle":"","mid":"0","fname":"Flume","special_id":"0","bak_id":"0","info":"0","hits":"514","pages":"1","comments":"0","posttime":"2019-05-16 02:08:03","list":"1557943683","username":"admin","author":"","copyfrom":"","copyfromurl":"","titlecolor":"","fonttype":"0","titleicon":"0","picurl":"https:\/\/www.cppentry.com\/upload_files\/","ispic":"0","yz":"1","yzer":"","yztime":"0","levels":"0","levelstime":"0","keywords":"Flume<\/A> \u5b9a\u4e49<\/A> source<\/A>  <\/A> SQLSource<\/A> \u8f6c\u4e3a<\/A> json<\/A> \u683c\u5f0f<\/A>","jumpurl":"","iframeurl":"","style":"","template":"a:3:{s:4:\"foot\";s:0:\"\";s:8:\"bencandy\";s:0:\"\";s:4:\"head\";s:0:\"\";}","target":"0","ip":"47.106.78.186","lastfid":"0","money":"0","buyuser":"","passwd":"","allowdown":"","allowview":"","editer":"","edittime":"0","begintime":"0","endtime":"0","description":"                                  \u7248\u6743\u58f0\u660e\uff1a\u672c\u6587\u4e3a\u535a\u4e3b\u539f\u521b\u6587\u7ae0\uff0c\u672a\u7ecf\u535a\u4e3b\u5141\u8bb8\u4e0d\u5f97\u8f6c\u8f7d\u3002          https:\/\/blog.csdn.net\/github_37643896\/article\/details\/87258584                                                                        \u4e2a\u4eba\u4f7f\u7528flume \u76f8..","lastview":"1714040454","digg_num":"0","digg_time":"0","forbidcomment":"0","ifvote":"0","heart":"","htmlname":"","city_id":"0"},"page":"1"}