无法使用请求从 zillow 抓取自定义属性链接

Question

我正在尝试解析当我 select 来自 zillow 的两个下拉列表时填充的不同属性链接。 选择完选项后，我可以在开发工具的 json 中看到结果。 然而，当我使用下面的脚本做同样的事情时，我得到了一些奇怪的文本。

手动操作：

1. 导航到该站点
2. 从第一个下拉列表中选择选项
3. 从第二个下拉列表中选择选项

这就是我尝试自动化的方式：

import json
import requests
from pprint import pprint

link = 'https://www.zillow.com/search/GetSearchPageState.htm?'

params = {
    'searchQueryState': {"pagination":{},"usersSearchTerm":"Vista, CA","mapBounds":{"west":-117.44051346728516,"east":-116.99488053271484,"south":33.126944633035116,"north":33.27919773006566},"regionSelection":[{"regionId":41517,"regionType":6}],"isMapVisible":True,"filterState":{"doz":{"value":"6m"},"isForSaleByAgent":{"value":False},"isForSaleByOwner":{"value":False},"isNewConstruction":{"value":False},"isForSaleForeclosure":{"value":False},"isComingSoon":{"value":False},"isAuction":{"value":False},"isPreMarketForeclosure":{"value":False},"isPreMarketPreForeclosure":{"value":False},"isRecentlySold":{"value":True},"isAllHomes":{"value":True},"hasPool":{"value":True},"hasAirConditioning":{"value":True},"isApartmentOrCondo":{"value":False}},"isListVisible":True,"mapZoom":11},
    'wants': {"cat1":["listResults","mapResults"]},
    'requestId': 2
}

with requests.Session() as s:
    s.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36'
    res = s.get(link,params=json.dumps(params))
    pprint(res.content)

这是它产生的 output：

b'<!-- This page outputs JSON instead of anything written here. -->'

如何使用请求解析来自 zillow 的自定义属性链接？

Answer 1

您必须对出现在request URL中的查询字符串进行编码。

为此，您需要：

urllib.parse.urlencode()

这是一个工作示例：

import json
import urllib.parse

import requests

link = 'https://www.zillow.com/search/GetSearchPageState.htm?'

params = {
    'searchQueryState': {
        "pagination": {},
        "usersSearchTerm": "Vista, CA",
        "mapBounds": {
            "west": -117.44051346728516,
            "east": -116.99488053271484,
            "south": 33.126944633035116,
            "north": 33.27919773006566
        },
        "regionSelection": [{"regionId": 41517, "regionType": 6}],
        "isMapVisible": True,
        "filterState": {
            "doz": {"value": "6m"}, "isForSaleByAgent": {"value": False},
            "isForSaleByOwner": {"value": False}, "isNewConstruction": {"value": False},
            "isForSaleForeclosure": {"value": False}, "isComingSoon": {"value": False},
            "isAuction": {"value": False}, "isPreMarketForeclosure": {"value": False},
            "isPreMarketPreForeclosure": {"value": False},
            "isRecentlySold": {"value": True}, "isAllHomes": {"value": True},
            "hasPool": {"value": True}, "hasAirConditioning": {"value": True},
            "isApartmentOrCondo": {"value": False}
        },
        "isListVisible": True,
        "mapZoom": 11
    },
    'wants': {"cat1": ["listResults"]},
    'requestId': 2
}

with requests.Session() as s:
    s.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36'
    s.headers["x-requested-session"] = "BE6D8DA620E60010D84B55EB18DC9DC8"
    s.headers["cookie"] = f"JSESSIONID={s.headers['x-requested-session']}"
    data = json.dumps(
        json.loads(s.get(f"{link}{urllib.parse.urlencode(params)}").content),
        indent=2
    )
    print(data)

Output：

{
  "user": {
    "isLoggedIn": false,
    "hasHousingConnectorPermission": false,
    "savedSearchCount": 0,
    "savedHomesCount": 0,
    "personalizedSearchGaDataTag": null,
    "personalizedSearchTraceID": "607a9ecb5aabe489c361c1d91f368b37",
    "searchPageRenderedCount": 0,
    "guid": "33b7add3-bfd3-4d85-a88a-d9d99256d2a2",
    "zuid": "",
    "isBot": false,
    "userSpecializedSEORegion": false
  },
  "mapState": {
    "customRegionPolygonWkt": null,
    "schoolPolygonWkt": null,
    "isCurrentLocationSearch": false,
    "userPosition": {
      "lat": null,
      "lon": null
    },
    "regionBounds": {
      "north": 33.275284,
      "east": -117.145153,
      "south": 33.130865,
      "west": -117.290241
    }
  },

and much much more ...

注意： Go 在那个网站上很容易，因为他们有非常敏感的反机器人措施，如果你继续请求数据太快，他们会向你扔一个验证码。