Python web 抓取错误“请启用 JavaScript 以查看页面内容。”

Question

当我尝试抓取相关网站时，我收到这样的错误消息，我不知道该怎么办。 我用 selenium 尝试了同一个站点，但不幸的是结果是一样的

import requests

headers = {
    'authority': 'www.beinconnect.com.tr',
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'accept-language': 'tr',
    'cookie': '_n_did=0d41e97b-f73a-47a2-9015-5a8313094ef1; _n_iid=oAXZXyYkc3; _fbp=fb.2.1644249804625.1072522209; InformationHash=; ai_user=U7uvw|2022-04-12T15:23:45.661Z; _cc_id=cfc8d1641d18649f974f5f7e71c76a32; _clck=zlka5i|1|f0n|0; .analyticsId=SA-19255803; .gapl={"MemberSource":"Turksat","CustomerSegment":"STANDARD"}; .dzdy=EED63C9FD637A760B04CCEBB14147A7A957BC72F555C669FDE85C20F35582650BFBD257190083C06B71556FEBE658DDDE97C6BD6B009559379C4AA414D03EE29D02E90C33E5DB8E46CD5F742FB35ECF12CFFFB7F8247689980CFB39B194DE37893F0143BC449EE1B506F95DFD0F1282175DA8D6B474FA998E76D592D0A63498148980976165B2A280A0E22DAA9551D3C74DEF5CF88766DEF4C333F2F5479D760AF82DEF939002B697EA09DA1A1190B69C0DE65CD9F85F7F2BFBFE501054500952754F8F60416F181553A54D0BFA4338A5456BF0A927979BBDDFA649DF52C6376F01A2FE5A6675E76AC67C733783147B0D3FA7A64F2CDAE4F27DB8549556EE2755B9F201E4E626DA05DA0BD0999ECCB9936E1859D56897F9BB93743F18ECF2A2B04D126879F7691E08B655AE973ACB7430EBBCAAC35E5F3F3895BA02A8B90704409B09A47918408B94516E7A8293F48FE3521A868BA58EE939B96BAA455C688353C52EE7E509AC6B9E8432C33C6AD5128288E323A57E8D1706073EC3449FC50E3D84C6A6F53B94EA31FB637E84F21CA4F11F0A0CB92F3CBFC8B1C5669BC68F9CC71BBAF46AF39DC712F055735A5712F82D77D0A9C6C915B7724A92D9DE3D8F365BBD586CCBACCE12D4BC0BBDC8761CD4A804BAF079C68F1478744E41A303782DB150E6B8795D06F4195039A127FC87C9971D6C428CF0D89BFF9984FFBD42D7FF95F29534B814E8A2E04AFFF7ABE6678163BC210D6DF2583CD9191191997650AF3325EAAF40307B664E24051C974552E36AC37EFA613072AB3BC0A2142B8CC59569FA2099A2C81F5DEB61F9262072C18AA3E07C402C9261A912644BB939962E7E094619DF0D75671E79F3DDA7473701D1C939975BC0CF3A5EC620158BD949D892327C89C3C065D1D594D532476D6FEE1F1A8A1667D4C75516CBEBE47E3E99060BE222564D22DF389392A2D0262B3366835A30A4580EE55F69BB3C84420A91F4170B04716F4126D0B2F239243FA7519E80ED68094BAEC0F409EE69623491EF70A5722EEED324E0E1B092CC37440F0A1414457855F55386144661DC41BA794EF557D04A1034EF371092B7452F0A9466631C91F72342586B4E7FF2A1847BF0FC73B59320C8AFD42BAD86B7A8220CE9BC67DA3CA61DA3E59CB4D1BC26D2392D4B8AF587F1FBAE6C10399B0849EB62D0434F9D14528B9B6DC6526D36A428A7DC0E17A2D38490F20B99C2020379BB28CB4E40F82590261A404B142E3572776A54446DA6B228F341FF15B4EF6D31A70BD40C42E7136E790C6D2C6A45EDE5AFC123DCB6BE76739D316782AF3D67F01231F6D98C64EA2FBA702ED320FDD5E3E76DE039CA51CEBB466FE; AccountNo=5053755679; _n_xid=SA-19255803; _n_uid=3Ofmm8EcSYDRi; _gcl_au=1.1.1719125645.1659859221; _gid=GA1.3.1198599326.1663777289; __RequestVerificationToken=dUJUfUhk5sjg50qLZRz0mopsCp5fd2sncEajkR9otaGIyd3Aq5ZeZ5779D0FO7Od8oaOoBXhnHkoxC_192hhuG5GKUU1; .c_ms=82A1233DB6B8431C8B59BBFF32B6DB22C4C5272567DA42B59F049DF7D87B20F5; .lctng=9BFPPG3jZ7zRkIY498IWqHEzsXtU6ZR7DAtAmQDhZ5JvTLybyTzG15XUNokjOG0Tu4F/sg==; panoramaId_expiry=1664369381823; TSPD_101=08a7fcea39ab28007889d7582264486a87ac40ea0e67c29fc01b7716f1442273da70ae82e38ce2293c1cb321a8051d1608152ca52a0518003d593ab179eb4fc37984acf5da0ec420ee249bcb083f6ac8; TS016b5396=011d35fd7b10b2f2f7b6b3c15231d422fd3ee622351fcb3f5c0aaf693d6bf2dfe115566f173c1fce2150f098cbcb0c6dbb10720a92; _ga_MX21GRT0FN=GS1.1.1664290807.86.1.1664290812.55.0.0; _ga=GA1.1.1165335072.1644249802; cto_bundle=gkXy_F9sMHpTNnJNYTNsOHpsYVJ2emVSeXo1NVJidkRTYVdDTkZRSzIwVnY5UTVZYTFLUE5sYkFUbzJQUFg4SSUyRjMzOGpDTXo4UnJTdWUxMlh6MnRsT002ZVpPYk1uYzlWd0V4cmh6JTJCdkxxaGFBdEN5aGg3ZkRiU1NScndHcGY0dGFVRUtBRjV6YkRhOWpncW9GdjRtUjBwbTdGZGNaQ1RpOE9hT0ZHQ2FzenNSd0YwJTNE; TS087214e8027=08a7fcea39ab200059738b9e802717f9a543966a278edb35ba2be0ae9405042163f58e43f442caee08552d593411300090ef62482f210251b466687d59065f63640585803a277d67039c8367c2518e47d9bd7eb060a93081235514f2a3a05adb; TS9b248c88077=08a7fcea39ab2800bfe66a4e59336a0b0196f9fa42b98aba39af7f730d6a840252a8870ccf15578a53daf00e98a8ad820827763fd01720005342854299b30751c8e3e1e94fb34104e3a013926d132f442f37469037b7d0db; ai_session=6sHVu|1664290807692|1664291757948.3',
    'referer': 'https://www.beinconnect.com.tr/film/scream-2022',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36',
    }

r = requests.get('https://www.beinconnect.com.tr/film/scream-2022', headers=headers)

print(r.text)

from selenium import webdriver

DRIVER_PATH = 'chromedriver.exe'
driver = webdriver.Chrome(executable_path=DRIVER_PATH)
driver.get('https://www.beinconnect.com.tr/film/scream-2022')
print(driver.page_source)

这是我找不到解决方案的麻烦错误响应。

<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<HTML>
  <HEAD>
    <META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
    <TITLE>ERROR: The request could not be satisfied</TITLE>
  </HEAD>
  <BODY>
    <H1>502 ERROR</H1>
    <H2>The request could not be satisfied.</H2>
    <HR noshade size="1px"> The origin closed the connection. We can't connect to the server for this app or website at this time. There might be too much traffic or a configuration error. Try again later, or contact the app or website owner. <BR clear="all"> If you provide content to customers through CloudFront, you can find steps to troubleshoot and help prevent this error by reviewing the CloudFront documentation. <BR clear="all">
    <HR noshade size="1px">
    <PRE>
Generated by cloudfront (CloudFront)
Request ID: uBM5jkz28D6-yM8C0EW7_XjccWiJVX_VMq-1ec0k5vKB_3Ll7q6q0w==
</PRE>
    <ADDRESS></ADDRESS>
  </BODY>
</HTML>

同一个请求另一个响应

<!DOCTYPE html>

<html><head>

<meta http-equiv="Pragma" content="no-cache"/>

<meta http-equiv="Expires" content="-1"/>

<meta http-equiv="CacheControl" content="no-cache"/>

<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>

<link rel="shortcut icon" href="data:;base64,iVBORw0KGgo="/>



<script type="text/javascript">

(function(){

window["bobcmn"] = "1011111010101020000000520000000520000000620000000129b248c8820000009620000000020000000230000000030000003bhttps%3a%2f%2fwww.beinconnect.com.tr%2ffilm%2fscream%2d2022300000006/TSPD/300000008TSPD_10130000000cTSPD_101_DID300000005https3000000b008a7fcea39ab20008f1b9ec3d9f40357c721948ec8e5fff911f61435e99a56c5527f5458c72a402308859ce9680a28003c6a0596547a7b5df1f49f73f251b6945b357d0b6fdc79d1c5091ba0fd94da40cf4db6bc010cf210300000002TS200000000200000000";



window["failureConfig"] = "524f6f70732e2e2e2e736f6d657468696e672077656e742077726f6e672e2e2e2e20796f757220737570706f72742069642069733a2025444f534c372e6368616c6c656e67652e737570706f72745f6964252e142038343836343131323931363530393039343932062f545350442f171801";window.Gbur=!!window.Gbur;try{(function(){(function(){})();var ji=16;try{var li,Oi,Zi=J(982)?0:1,_i=J(880)?0:1,II=J(611)?0:1,lI=J(920)?0:1,LI=J(558)?0:1,ZI=J(288)?0:1,SI=J(991)?0:1,_I=J(430)?0:1;for(var Ij=(J(888),0);Ij<Oi;++Ij)Zi+=(J(974),2),_i+=J(634)?1:2,II+=(J(868),2),lI+=(J(900),2),LI+=J(55)?2:1,ZI+=J(351)?1:2,SI+=(J(487),2),_I+=J(255)?2:3;li=Zi+_i+II+lI+LI+ZI+SI+_I;window.Ji===li&&(window.Ji=++li)}catch(jj){window.Ji=li}var Lj=!0;
function L(I){var l=arguments.length,O=[],s=1;while(s<l)O[s-1]=arguments[s++]-I;return String.fromCharCode.apply(String,O)}function oj(I){var l=25;!I||document[z(l,143,130,140,130,123,130,133,130,141,146,108,141,122,141,126)]&&document[z(l,143,130,140,130,123,130,133,130,141,146,108,141,122,141,126)]!==Z(68616527641,l)||(Lj=!1);return Lj}function z(I){var l=arguments.length,O=[];for(var s=1;s<l;++s)O.push(arguments[s]-I);return String.fromCharCode.apply(String,O)}function Oj(){}
oj(window[Oj[L(ji,126,113,125,117)]]===Oj);oj(typeof ie9rgb4!==Z(1242178186183,ji));oj(RegExp("\x3c")[Z(1372189,ji)](function(){return"\x3c"})&!RegExp(z(ji,136,67,116))[z(ji,132,117,131,132)](function(){return"'x3'+'d';"}));
var Zj=window[L(ji,113,132,132,113,115,120,85,134,117,126,132)]||RegExp(z(ji,125,127,114,121,140,113,126,116,130,127,121,116),Z(2,ji))[Z(1372189,ji)](window["\x6e\x61vi\x67a\x74\x6f\x72"]["\x75\x73e\x72A\x67\x65\x6et"]),sj=+new Date+(J(342)?465449:6E5),iJ,jJ,JJ,LJ=window[L(ji,131,117,132,100,121,125,117,127,133,132)],oJ=Zj?J(777)?30260:3E4:J(899)?8077:6E3;
document[z(ji,113,116,116,85,134,117,126,132,92,121,131,132,117,126,117,130)]&&document[L(ji,113,116,116,85,134,117,126,132,92,121,131,132,117,126,117,130)](z(ji,134,121,131,121,114,121,124,121,132,137,115,120,113,126,119,117),function(I){var l=91;document[z(l,209,196,206,196,189,196,199,196,207,212,174,207,188,207,192)]&&(document[L(l,209,196,206,196,189,196,199,196,207,212,174,207,188,207,192)]===z(l,195,196,191,191,192,201)&&I[z(l,196,206,175,205,208,206,207,192,191)]?JJ=!0:document[z(l,209,196,
206,196,189,196,199,196,207,212,174,207,188,207,192)]===Z(68616527575,l)&&(iJ=+new Date,JJ=!1,OJ()))});function OJ(){if(!document[z(85,198,202,186,199,206,168,186,193,186,184,201,196,199)])return!0;var I=+new Date;if(I>sj&&(J(835)?314419:6E5)>I-iJ)return oj(!1);var l=oj(jJ&&!JJ&&iJ+oJ<I);iJ=I;jJ||(jJ=!0,LJ(function(){jJ=!1},J(140)?0:1));return l}OJ();var ZJ=[J(459)?14863572:17795081,J(861)?2147483647:27611931586,J(901)?1706666747:1558153217];function Z(I,l){I+=l;return I.toString(36)}
function sJ(I){var l=67;I=typeof I===L(l,182,183,181,172,177,170)?I:I[L(l,183,178,150,183,181,172,177,170)](J(236)?47:36);var O=window[I];if(!O||!O[z(l,183,178,150,183,181,172,177,170)])return;var s=""+O;window[I]=function(I,l){jJ=!1;return O(I,l)};window[I][z(l,183,178,150,183,181,172,177,170)]=function(){return s}}for(var _J=(J(230),0);_J<ZJ[Z(1294399189,ji)];++_J)sJ(ZJ[_J]);oj(!1!==window[z(ji,87,114,133,130)]);window.l_=window.l_||{};window.l_._L="086d58930a0180001f25c941f40f731e3d734e5def41fa229a71fcab40281f87d2847512465e238b3d621c2ede661bbfab5d44e0e610c207690b86c5571c12919388db1f9c71535ee43541f485fa1d7f4e804b181780191177a21bde0dec7ac8379c458cbc304d502c8226d3c3a5d340e178a22e4e0281923f1b1e951ff50416c2dd3c85f2cdd8f0";
function Il(I){var l=+new Date,O;!document[L(73,186,190,174,187,194,156,174,181,174,172,189,184,187,138,181,181)]||l>sj&&(J(86)?395124:6E5)>l-iJ?O=oj(!1):(O=oj(jJ&&!JJ&&iJ+oJ<l),iJ=l,jJ||(jJ=!0,LJ(function(){jJ=!1},J(610)?0:1)));return!(arguments[I]^O)}function J(I){return 81>I}(function(){var I=/(\A([0-9a-f]{1,4}:){1,6}(:[0-9a-f]{1,4}){1,1}\Z)|(\A(([0-9a-f]{1,4}:){1,7}|:):\Z)|(\A:(:[0-9a-f]{1,4}){1,7}\Z)/ig,l=document.getElementsByTagName("head")[0],O=[];l&&(l=l.innerHTML.slice(0,1E3));while(l=I.exec(""))O.push(l)})();})();}catch(x){}finally{ie9rgb4=void(0);};function ie9rgb4(a,b){return a>>b>>0};


})();

</script>

<script type="text/javascript" src="/TSPD/08a7fcea39ab2000aa3cc9facafa33e817bc3152a19e0e14f824263084b9be17af1a47513bfa23a3?type=10"></script>

<noscript>Please enable JavaScript to view the page content.<br/>Your support ID is:  8486411291650909492.<link rel="stylesheet" href="/TSPD/?type=25" " /></noscript>

</head><body>

</body></html>

欢迎任何帮助，因为我不太擅长这个。

Answer 1

这似乎现在可以工作，但我需要让它静默运行，我会继续尝试不同的模块，谢谢你的提示。

如果有不同的代码，我会留下这个问题。

from selenium import webdriver
import time

url = "https://www.beinconnect.com.tr/film/scream-2022"

driver = webdriver.Chrome(executable_path='chromedriver')
driver.get(url)

time.sleep(5)

html = driver.page_source
print(html)