python爬虫实例:亚马逊商品页面的爬取
laiczhang
2019年05月28日 17:24
收录于文集
共16篇

先运行如下代码:

import requests

r=requests.get("https://www.amazon.cn/gp/product/B01M8L5Z3Y")

r.status_code

运行结果为:

200

这部分结果和老师的就有所不同了,老师那边返回的是503,查看结果之后发现是代理user-agent了问题,所以对user-agent进行了设置,这里两种均尝试下

方法1:

#未修改user-agent

import requests

url="https://www.amazon.cn/gp/product/B01M8L5Z3Y"

try:

    r=requests.get(url)

    r.raise_for_status()

    r.encoding=r.apparent_encoding

    print(r.text[2000:3000])

except:

    print("爬取失败&#​34;)

运行结果为:

ges-cn.ssl-images-amazon.com/images/G/28/javascripts/lib/popover/images/po_top_left._V1_.png)}.ap_popover_unsprited .ap_header .ap_right{background-image:url(https://images-cn.ssl-images-amazon.com/images/G/28/javascripts/lib/popover/images/po_top_right._V1_.png)}.ap_popover_unsprited .ap_header .ap_middle{background-image:url(https://images-cn.ssl-images-amazon.com/images/G/28/javascripts/lib/popover/images/po_top._V1_.png)}.ap_popover_unsprited .ap_footer .ap_left{background-image:url(https://images-cn.ssl-images-amazon.com/images/G/28/javascripts/lib/popover/images/po_bottom_left._V1_.png)}.ap_popover_unsprited .ap_footer .ap_right{background-image:url(https://images-cn.ssl-images-amazon.com/images/G/28/javascripts/lib/popover/images/po_bottom_right._V1_.png)}.ap_popover_unsprited .ap_footer .ap_middle{background-image:url(https://images-cn.ssl-images-amazon.com/images/G/28/javascripts/lib/popover/images/po_bottom._V1_.png)}.ap_popover_sprited .ap_body .ap_left,.ap_popover_sprited .

方法2:

#修改user-agent

import requests

url="https://www.amazon.cn/gp/product/B01M8L5Z3Y"

try:

    kv={'user-agent&#​39;:'Mozilla/5.0&#​39;}

    r=requests.get(url,headers=kv)

    r.raise_for_status()

    r.encoding=r.apparent_encoding

    print(r.text[2000:3000])

except:

    print("爬取失败&#​34;)

运行结果为:

om.amazon.csm.nexusclient.prod',

ue_navtiming=1,

ue_fcsn=1,

ue_isrw=true,

ue_fpf='//fls-cn.amazon.cn/1/batch/1/OP/AAHKV2X7AFYLW:462-6324598-2062068:EE3CSP6EGPZB02RRZ8RQ$uedata=s:',

ue_qsl=2000,

ue_rpl_ns=0,

ue_orct=1,

ue_int=0,

ue_adb=1,

ue_adb_rtla=1,

ue_ddq=1,

ue_sspb=0,

ue_rsc=0,

ue_clf=0,

ue_cdt=0,

ue_pel=0,

ue_wdg_imp=0;

if (!window.ue_csm) {var ue_csm = window;}

function ue_viz(){(function(c,e,a){function k(b){if(c.ue.viz.length<p&&!l){var a=b.type;b=b.originalEvent;/^focus./.test(a)&&b&&(b.toElement||b.fromElement||b.relatedTarget)||(a=e[m]||("blur&#​34;==a||"focusout&#​34;==a?"hidden&#​34;:"visible&#​34;),c.ue.viz.push(a+":&#​34;+(+new Date-c.ue.t0)),"visible&#​34;==a&&(ue.isl&&uex("at&#​34;),l=1))}}for(var l=0,f,g,m,n=["&#​34;,"webkit&#​34;,"o&#​34;,"ms&#​34;,"moz&#​34;],d=0,p=20,h=0;h<n.length&&!d;h++)if(a=n[h],f=(a?a+"H&#​34;:"h&#​34;)+"idden&#​34;,d="boolean&#​34;==typeof e[f])g=a+"visibilitychange&#​34;,m=(a?a+"V&#​34;:"v&#​34;)+"isibilityState&#​34;;

k({});d&&e.addEventListener(g,k,0);c.ue&&d&&(c.ue.pageViz={event:g,propHid:f})})(ue_csm,document,window)};

(function a,

这部分和之前结果竟然不一样,也是挺神奇的,后续再探索原因。