{"rsdb":{"rid":"372442","subhead":"","postdate":"0","aid":"261446","fid":"77","uid":"1","topic":"1","content":"<div id=\"cnblogs_post_body\" class=\"blogpost-body \"> \n <h1>\u524d\u8a00<\/h1> \n <p>\u95f2\u6765\u65e0\u4e8b\u5c31\u8981\u7ec3\u7ec3\u4ee3\u7801\uff0c\u4e0d\u77e5\u9053\u6700\u8fd1\u722c\u53d6\u4ec0\u4e48\u7f51\u7ad9\u597d\uff0c\u5c31\u62ff\u7eb5\u6a2a\u7f51\u722c\u53d6\u6211\u6700\u559c\u6b22\u7684\u96ea\u4e2d\u608d\u5200\u884c\u7ec3\u624b\u5427<\/p> \n <h2>\u51c6\u5907<\/h2> \n <ul> \n  <li><a href=https:\/\/www.cppentry.com style=text-decoration:underline;font-size:14px;color:#F70968; target=_blank>python<\/a>3<\/li> \n  <li>scrapy<\/li> \n <\/ul> \n <h2>\u9879\u76ee\u521b\u5efa\uff1a<\/h2> \n <p>cmd\u547d\u4ee4\u884c\u5207\u6362\u5230\u5de5\u4f5c\u76ee\u5f55\u521b\u5efascrapy\u9879\u76ee&nbsp; \u4e24\u6761\u547d\u4ee4 scarpy startproject\u4e0escrapy genspider&nbsp; &nbsp;&nbsp;\u7136\u540e\u7528pycharm\u6253\u5f00\u9879\u76ee<\/p> \n <div class=\"cnblogs_code\"> \n  <pre>D:\\<a href=https:\/\/www.cppentry.com style=text-decoration:underline;font-size:14px;color:#F70968; target=_blank>python<\/a>work&gt;<span style=\"color: #000000;\"><span style=\"color: #ff0000;\">scrapy startproject zongheng<\/span>\r\nNew Scrapy project <\/span><span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">zongheng<\/span><span style=\"color: #800000;\">'<\/span>, using template directory <span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">c:\\users\\11573\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\scrapy\\templates\\project<\/span><span style=\"color: #800000;\">'<\/span>, created <span style=\"color: #0000ff;\">in<\/span><span style=\"color: #000000;\">:\r\n    D:\\pythonwork\\zongheng\r\n\r\nYou can start your first spider with:\r\n    cd zongheng\r\n    scrapy genspider example example.com\r\n\r\nD:\\pythonwork<\/span>&gt;<span style=\"color: #000000;\">cd zongheng\r\n\r\nD:\\pythonwork\\zongheng<\/span>&gt;<span style=\"color: #000000;\">cd zongheng\r\n\r\nD:\\pythonwork\\zongheng\\zongheng<\/span>&gt;<span style=\"color: #ff0000;\">scrapy genspider xuezhong http:\/\/book.zongheng.com\/chapter\/189169\/3431546.html<\/span>\r\nCreated spider <span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">xuezhong<\/span><span style=\"color: #800000;\">'<\/span> using template <span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">basic<\/span><span style=\"color: #800000;\">'<\/span> <span style=\"color: #0000ff;\">in<\/span><span style=\"color: #000000;\"> module:\r\n  zongheng.spiders.xuezhong<\/span><\/pre> \n <\/div> \n <hr \/> \n <h2>\u786e\u5b9a\u5185\u5bb9<\/h2> \n <p>\u9996\u5148\u6253\u5f00\u7f51\u9875\u770b\u4e0b\u6211\u4eec\u9700\u8981\u722c\u53d6\u7684\u5185\u5bb9<\/p> \n <p><img src=\"https:\/\/img2018.cnblogs.com\/blog\/1699970\/201909\/1699970-20190918141003377-1758620514.png\" alt=\"\" width=\"923\" height=\"513\" \/><\/p> \n <p>\u5176\u5b9e\u5c0f\u8bf4\u7684\u8bdd\u7ed3\u6784\u6bd4\u8f83\u7b80\u5355 \u53ea\u6709\u4e09\u5927\u5757&nbsp; \u5377 \u7ae0\u8282 \u5185\u5bb9<\/p> \n <p>\u56e0\u6b64 items.py\u4ee3\u7801\uff1a<\/p> \n <div class=\"cnblogs_code\"> \n  <pre><span style=\"color: #008000;\">#<\/span><span style=\"color: #008000;\"> -*- coding: utf-8 -*-<\/span>\r\n\r\n<span style=\"color: #008000;\">#<\/span><span style=\"color: #008000;\"> Define here the models for your scraped items<\/span><span style=\"color: #008000;\">\r\n#\r\n#<\/span><span style=\"color: #008000;\"> See documentation in:<\/span><span style=\"color: #008000;\">\r\n#<\/span><span style=\"color: #008000;\"> https:\/\/docs.scrapy.org\/en\/latest\/topics\/items.html<\/span>\r\n\r\n<span style=\"color: #0000ff;\">import<\/span><span style=\"color: #000000;\"> scrapy\r\n\r\n\r\n<\/span><span style=\"color: #0000ff;\">class<\/span><span style=\"color: #000000;\"> ZonghengItem(scrapy.Item):\r\n    <\/span><span style=\"color: #008000;\">#<\/span><span style=\"color: #008000;\"> define the fields for your item here like:<\/span>\r\n    <span style=\"color: #008000;\">#<\/span><span style=\"color: #008000;\"> name = scrapy.Field()<\/span>\r\n    book =<span style=\"color: #000000;\"> scrapy.Field()\r\n    section <\/span>=<span style=\"color: #000000;\"> scrapy.Field()\r\n    content <\/span>=<span style=\"color: #000000;\"> scrapy.Field()\r\n    <\/span><span style=\"color: #0000ff;\">pass<\/span><\/pre> \n <\/div> \n <hr \/> \n <h2>\u5185\u5bb9\u63d0\u53d6spider\u6587\u4ef6\u7f16\u5199<\/h2> \n <p>\u8fd8\u662f\u6211\u4eec\u5148\u521b\u5efa\u4e00\u4e2amain.py\u6587\u4ef6\u65b9\u4fbf\u6211\u4eec\u6d4b\u8bd5\u4ee3\u7801<\/p> \n <div class=\"cnblogs_code\"> \n  <pre><span style=\"color: #0000ff;\">from<\/span> scrapy <span style=\"color: #0000ff;\">import<\/span><span style=\"color: #000000;\"> cmdline\r\ncmdline.execute(<\/span><span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">scrapy crawl xuezhong<\/span><span style=\"color: #800000;\">'<\/span>.split())<\/pre> \n <\/div> \n <p>\u7136\u540e\u6211\u4eec\u53ef\u4ee5\u5728spider\u6587\u4ef6\u4e2d\u5148\u7f16\u5199<\/p> \n <div class=\"cnblogs_code\"> \n  <pre><span style=\"color: #008000;\">#<\/span><span style=\"color: #008000;\"> -*- coding: utf-8 -*-<\/span>\r\n<span style=\"color: #0000ff;\">import<\/span><span style=\"color: #000000;\"> scrapy\r\n\r\n\r\n<\/span><span style=\"color: #0000ff;\">class<\/span><span style=\"color: #000000;\"> XuezhongSpider(scrapy.Spider):\r\n    name <\/span>= <span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">xuezhong<\/span><span style=\"color: #800000;\">'<\/span><span style=\"color: #000000;\">\r\n    allowed_domains <\/span>= [<span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">http:\/\/book.zongheng.com\/chapter\/189169\/3431546.html<\/span><span style=\"color: #800000;\">'<\/span><span style=\"color: #000000;\">]\r\n    start_urls <\/span>= [<span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">http:\/\/book.zongheng.com\/chapter\/189169\/3431546.html\/<\/span><span style=\"color: #800000;\">'<\/span><span style=\"color: #000000;\">]\r\n\r\n    <\/span><span style=\"color: #0000ff;\">def<\/span><span style=\"color: #000000;\"> parse(self, response):\r\n        <\/span><span style=\"color: #0000ff;\">print<\/span><span style=\"color: #000000;\">(response.text)\r\n        <\/span><span style=\"color: #0000ff;\">pass<\/span><\/pre> \n <\/div> \n <p>\u8fd0\u884cmain.py\u770b\u770b\u6709\u6ca1\u6709\u8f93\u51fa<\/p> \n <p>\u53d1\u73b0\u76f4\u63a5\u6574\u4e2a\u7f51\u9875\u7684\u5185\u5bb9\u90fd\u53ef\u4ee5\u722c\u53d6\u4e0b\u6765\uff0c\u8bf4\u660e\u8be5\u7f51\u9875\u57fa\u672c\u6ca1\u6709\u53cd\u722c\u673a\u5236\uff0c\u751a\u81f3\u4e0d\u7528\u6211\u4eec\u53bb\u4fee\u6539user-agent\u90a3\u4e48\u5c31\u76f4\u63a5\u5f00\u59cb\u5427<\/p> \n <p>\u6253\u5f00\u7f51\u9875 F12\u67e5\u770b\u5143\u7d20\u4f4d\u7f6e \u5e76\u7f16\u5199xpath\u8def\u5f84 \u7136\u540e\u7f16\u5199spider\u6587\u4ef6<\/p> \n <p>\u9700\u8981\u6ce8\u610f\u7684\u662f<span style=\"color: #ff0000;\">\u6211\u4eec\u8981\u5bf9\u5c0f\u8bf4\u5185\u5bb9\u8fdb\u884c\u4e00\u5b9a\u91cf\u7684\u6570\u636e\u6e05\u6d17\uff0c\u56e0\u4e3a\u5305\u542b\u67d0\u4e9bhtml\u6807\u7b7e\u6211\u4eec\u9700\u8981\u53bb\u9664<\/span><\/p> \n <div class=\"cnblogs_code\"> \n  <pre><span style=\"color: #008000;\">#<\/span><span style=\"color: #008000;\"> -*- coding: utf-8 -*-<\/span>\r\n<span style=\"color: #0000ff;\">import<\/span><span style=\"color: #000000;\"> scrapy\r\n<\/span><span style=\"color: #0000ff;\">import<\/span><span style=\"color: #000000;\"> re\r\n<\/span><span style=\"color: #0000ff;\">from<\/span> zongheng.items <span style=\"color: #0000ff;\">import<\/span><span style=\"color: #000000;\"> ZonghengItem\r\n\r\n\r\n<\/span><span style=\"color: #0000ff;\">class<\/span><span style=\"color: #000000;\"> XuezhongSpider(scrapy.Spider):\r\n    name <\/span>= <span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">xuezhong<\/span><span style=\"color: #800000;\">'<\/span><span style=\"color: #000000;\">\r\n    allowed_domains <\/span>= [<span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">book.zongheng.com<\/span><span style=\"color: #800000;\">'<\/span><span style=\"color: #000000;\">]\r\n    start_urls <\/span>= [<span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">http:\/\/book.zongheng.com\/chapter\/189169\/3431546.html\/<\/span><span style=\"color: #800000;\">'<\/span><span style=\"color: #000000;\">]\r\n\r\n    <\/span><span style=\"color: #0000ff;\">def<\/span><span style=\"color: #000000;\"> parse(self, response):\r\n        xuezhong_item <\/span>=<span style=\"color: #000000;\"> ZonghengItem()\r\n        xuezhong_item[<\/span><span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">book<\/span><span style=\"color: #800000;\">'<\/span>] = response.xpath(<span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">\/\/*[@id=&quot;reader_warp&quot;]\/div[2]\/text()[4]<\/span><span style=\"color: #800000;\">'<\/span>).get()[3<span style=\"color: #000000;\">:]\r\n        xuezhong_item[<\/span><span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">section<\/span><span style=\"color: #800000;\">'<\/span>] = response.xpath(<span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">\/\/*[@id=&quot;readerFt&quot;]\/div\/div[2]\/div[2]\/text()<\/span><span style=\"color: #800000;\">'<\/span><span style=\"color: #000000;\">).get()\r\n\r\n        content <\/span>= response.xpath(<span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">\/\/*[@id=&quot;readerFt&quot;]\/div\/div[5]<\/span><span style=\"color: #800000;\">'<\/span><span style=\"color: #000000;\">).get()\r\n        <\/span><span style=\"color: #008000;\">#<\/span><span style=\"color: #008000;\">content\u5185\u5bb9\u9700\u8981\u5904\u7406\u56e0\u4e3a\u4f1a\u663e\u793a&lt;p&gt;&lt;\/p&gt;\u6807\u7b7e\u548c&lt;div&gt;\u6807\u7b7e<\/span>\r\n        content = re.sub(r<span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">&lt;\/p&gt;<\/span><span style=\"color: #800000;\">'<\/span>, <span style=\"color: #800000;\">&quot;&quot;<\/span><span style=\"color: #000000;\">, content)\r\n        content <\/span>= re.sub(r<span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">&lt;p&gt;|&lt;div.*&gt;|&lt;\/div&gt;<\/span><span style=\"color: #800000;\">'<\/span>,<span style=\"color: #800000;\">&quot;<\/span><span style=\"color: #800000;\">\\n<\/span><span style=\"color: #800000;\">&quot;<\/span><span style=\"color: #000000;\">,content )\r\n\r\n        xuezhong_item[<\/span><span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">content<\/span><span style=\"color: #800000;\">'<\/span>] =<span style=\"color: #000000;\"> content\r\n        <\/span><span style=\"color: #0000ff;\">yield<\/span><span style=\"color: #000000;\"> xuezhong_item\r\n\r\n        nextlink <\/span>= response.xpath(<span style=\"color: #800000;\">'<\/span><span style=\"color: #800000;\">\/\/*[@id=&quot;readerFt&quot;]\/div\/div[7]\/a[3]\/@href<\/span><span style=\"color: #800000;\">'<\/span><span style=\"color: #000000;\">).get()\r\n        <\/span><span style=\"color: #0000ff;\">print<\/span><span style=\"color: #000000;\">(nextlink)\r\n        <\/span><span style=\"color: #0000ff;\">if<\/span><span style=\"color: #000000;\"> nextlink:\r\n            <\/span><span style=\"color: #0000ff;\">yield<\/span> scrapy.Request(nextlink,callback=self.parse)<\/pre> \n <\/div> \n <p>\u6709\u65f6\u5019\u6211\u4eec\u4f1a\u53d1\u73b0\u65e0\u6cd5\u8fdb\u5165\u4e0b\u4e2a\u94fe\u63a5\uff0c\u90a3\u53ef\u80fd\u662f\u88aballowed_domains\u8fc7\u6ee4\u6389\u4e86 \u6211\u4eec\u4fee\u6539\u4e0b\u5c31\u53ef\u4ee5<\/p> \n <p>\u5509 \u7a81\u7136\u53d1\u73b0\u4e86\u5230\u7b2c\u4e00\u5377\u7684\u4e00\u767e\u591a\u7ae0\u540e\u5c31\u8981VIP\u4e86 \u90a3\u6211\u4eec\u5c31\u5148\u53ea\u5f04\u4e00\u767e\u591a\u7ae0\u5427 \u4e0d\u8fc7\u4e5f\u53ef\u4ee5\u53bb\u5176\u4ed6\u7f51\u7ad9\u722c\u53d6\u514d\u8d39\u7684 \u8fd9\u6b21\u6211\u4eec\u5c31\u5148\u722c\u53d6\u4e00\u767e\u591a\u7ae0\u5427<\/p> \n <hr \/> \n <p>&nbsp;<\/p> \n <h2>\u5185\u5bb9\u4fdd\u5b58<\/h2> \n <p>\u63a5\u4e0b\u6765\u5c31\u662f\u5185\u5bb9\u7684\u4fdd\u5b58\u4e86\uff0c\u8fd9\u6b21\u5c31\u76f4\u63a5\u4fdd\u5b58\u4e3a\u672c\u5730txt\u6587\u4ef6\u5c31\u884c\u4e86&nbsp;<\/p> \n <p>\u9996\u5148\u53bbsettings.py\u6587\u4ef6\u91cc\u5f00\u542f&nbsp;&nbsp;ITEM_PIPELINES<\/p> \n <p>\u7136\u540e\u7f16\u5199pipelines.py\u6587\u4ef6<\/p> \n <div class=\"cnblogs_code\"> \n  <pre><span style=\"color: #008000;\">#<\/span><span style=\"color: #008000;\"> -*- coding: utf-8 -*-<\/span>\r\n\r\n<span style=\"color: #008000;\">#<\/span><span style=\"color: #008000;\"> D","orderid":"0","title":"scrapy \u722c\u53d6\u7eb5\u6a2a\u7f51\u5b9e\u6218(\u4e00)","smalltitle":"","mid":"0","fname":"Python","special_id":"0","bak_id":"0","info":"0","hits":"102","pages":"2","comments":"0","posttime":"2019-09-19 11:10:20","list":"1568862620","username":"admin","author":"","copyfrom":"","copyfromurl":"","titlecolor":"","fonttype":"0","titleicon":"0","picurl":"https:\/\/www.cppentry.com\/upload_files\/","ispic":"0","yz":"1","yzer":"","yztime":"0","levels":"0","levelstime":"0","keywords":"<A HREF='https:\/\/www.cppentry.com\/do\/search.php?type=keyword&keyword=scrapy' target=_blank>scrapy<\/A> <A HREF='https:\/\/www.cppentry.com\/do\/search.php?type=keyword&keyword=%D7%DD%BA%E1' target=_blank>\u7eb5\u6a2a<\/A> <A HREF='https:\/\/www.cppentry.com\/do\/search.php?type=keyword&keyword=%CA%B5%D5%BD' target=_blank>\u5b9e\u6218<\/A>","jumpurl":"","iframeurl":"","style":"","template":"a:3:{s:4:\"head\";s:0:\"\";s:4:\"foot\";s:0:\"\";s:8:\"bencandy\";s:0:\"\";}","target":"0","ip":"14.17.22.32","lastfid":"0","money":"0","buyuser":"","passwd":"","allowdown":"","allowview":"","editer":"","edittime":"0","begintime":"0","endtime":"0","description":"scrapy \u722c\u53d6\u7eb5\u6a2a\u7f51\u5b9e\u6218","lastview":"1714071197","digg_num":"0","digg_time":"0","forbidcomment":"0","ifvote":"0","heart":"","htmlname":"","city_id":"0"},"page":"1"}