这里有一个简单的
scrapy
任何人都可以用来测试的蜘蛛。
from scrapy.utils.response import open_in_browser
import scrapy
import json
class TestSpider(scrapy.Spider):
name = "test-spider"
allowed_domains = ["shopee.ph"]
shopee_cookies = '[{"name": "csrftoken", "value": "RvxBdTixvBfdTR3xfQwbcYippqz8jEbF", "domain": "shopee.ph", "path": "/", "expires": -1, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "_gcl_au", "value": "1.1.1251411089.1692464842", "domain": ".shopee.ph", "path": "/", "expires": 1700240842, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "SPC_SI", "value": "sTLbZAAAAABwY1ZrR1NNU+WdNgAAAAAAdzlCYXIyVVQ=", "domain": ".shopee.ph", "path": "/", "expires": 1692551246.336331, "httpOnly": true, "secure": true, "sameSite": "Lax"}, {"name": "_fbp", "value": "fb.1.1692464842990.689078803", "domain": ".shopee.ph", "path": "/", "expires": 1700240846, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "SPC_R_T_IV", "value": "NnVEbThnRjREMnNMZVpGVQ==", "domain": ".shopee.ph", "path": "/", "expires": 1727024846.336348, "httpOnly": false, "secure": true, "sameSite": "Lax"}, {"name": "SPC_T_ID", "value": "fn/OKngQO3doGdfFGyo/6mzLiviELHkKEbWM9J+x/ezTl/baT96grQer6ILrYX9tj3Kqs71Jg+hCimaK/XauidJXrd6HdPd2Smbxbu/fEStjOJi5g9/ucMmbBwuyh5M6H3TOGdpUop/9Q/zdpNj6MyxZaODnNsT5XprfsQxjB5g=", "domain": ".shopee.ph", "path": "/", "expires": 1727024846.336355, "httpOnly": true, "secure": true, "sameSite": "Lax"}, {"name": "SPC_T_IV", "value": "NnVEbThnRjREMnNMZVpGVQ==", "domain": ".shopee.ph", "path": "/", "expires": 1727024846.336362, "httpOnly": true, "secure": true, "sameSite": "Lax"}, {"name": "SPC_F", "value": "jiOtuCSNUaap3U4BHHfzhDihWwFht32f", "domain": ".shopee.ph", "path": "/", "expires": 1727024843.162052, "httpOnly": false, "secure": true, "sameSite": "Lax"}, {"name": "REC_T_ID", "value": "dc8a2570-3eb2-11ee-ac9b-2cea7fce6c95", "domain": ".shopee.ph", "path": "/", "expires": 1727024843.16206, "httpOnly": true, "secure": true, "sameSite": "Lax"}, {"name": "SPC_R_T_ID", "value": "fn/OKngQO3doGdfFGyo/6mzLiviELHkKEbWM9J+x/ezTl/baT96grQer6ILrYX9tj3Kqs71Jg+hCimaK/XauidJXrd6HdPd2Smbxbu/fEStjOJi5g9/ucMmbBwuyh5M6H3TOGdpUop/9Q/zdpNj6MyxZaODnNsT5XprfsQxjB5g=", "domain": ".shopee.ph", "path": "/", "expires": 1727024846.33634, "httpOnly": false, "secure": true, "sameSite": "Lax"}, {"name": "_QPWSDCXHZQA", "value": "4a585493-a7a0-4f0e-d696-687295d3a4c3", "domain": "shopee.ph", "path": "/", "expires": 1692496379, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "IDE", "value": "AHWqTUm1b5ZflCqDTn6cpHDjyoeqH6iLfXcCOOm4YNaP8CHTsAZ7F_Daq4-zO-bsGIk", "domain": ".doubleclick.net", "path": "/", "expires": 1727024843.787698, "httpOnly": true, "secure": true, "sameSite": "None"}, {"name": "AMP_TOKEN", "value": "%24NOT_FOUND", "domain": ".shopee.ph", "path": "/", "expires": 1692468444, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "_ga", "value": "GA1.2.833255521.1692464843", "domain": ".shopee.ph", "path": "/", "expires": 1727024844.498551, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "_gid", "value": "GA1.2.1347861977.1692464844", "domain": ".shopee.ph", "path": "/", "expires": 1692551244, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "_dc_gtm_UA-61918643-6", "value": "1", "domain": ".shopee.ph", "path": "/", "expires": 1692464904, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "shopee_webUnique_ccd", "value": "raj%2F3ukNopIWTrFjVLQeGA%3D%3D%7C1%2BjiV3ga9OlzuAELTZtedUY5BlP1ZNVH5ybZJx2D4KNA9dGTvtFakjnNZvR64zKNG6yBDfEXdabTE%2FRKow%3D%3D%7CsWIQ7u7pR4F3BD7E%7C08%7C3", "domain": "shopee.ph", "path": "/", "expires": 1692496381, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "ds", "value": "065598fda3b7cca4e5e241e446a075e9", "domain": "shopee.ph", "path": "/", "expires": 1692496381, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "SPC_EC", "value": "RTJYa2Q5WEV4UDNnN3VGWr68rFv1FRJEeVkpwAzlu09WhtwSxFE1cZlwpQYRhhR56REixPuKfekz6oioE4EaDK12bvALil+QZ5B0EfG42psIFWNDe1moiErTZndyu1502KUlh5+OQoUWCvm1XkVY+2Iy7Jk5qyPI2J655JeZwv0=", "domain": ".shopee.ph", "path": "/", "expires": 1727024846.336291, "httpOnly": true, "secure": true, "sameSite": "Lax"}, {"name": "SPC_ST", "value": ".ek1DVmo5aGJjaVBxcklYU5o4/3v/8ndPeV2/fwtzWYUh1kWOopWvn7SFoQXWuS37Rs+J+Ym7U8OwOG73JbiFRWyOOo1GhKBgwhUeeWfE+q9XPDZXACC33t7qphoBu5hyWvR/G+WkpSUbIkmGPzprCIvhw7Qwyt8UFxk/4bA+47QQQUiDcPfHIq/sJqmVMEqH3Al6nCTDeEh/JCDLALRvNQ==", "domain": ".shopee.ph", "path": "/", "expires": 1727024846.336324, "httpOnly": true, "secure": true, "sameSite": "Lax"}, {"name": "SPC_CLIENTID", "value": "amlPdHVDU05VYWFwgvlavxoisbqjmacw", "domain": ".shopee.ph", "path": "/", "expires": 1727024846.336374, "httpOnly": false, "secure": false, "sameSite": "Lax"}, {"name": "_ga_CB0044GVTM", "value": "GS1.1.1692464843.1.0.1692464846.57.0.0", "domain": ".shopee.ph", "path": "/", "expires": 1727024846.367333, "httpOnly": false, "secure": false, "sameSite": "Lax"}]'
shopee_cookies = json.loads(shopee_cookies)
def start_requests(self):
yield scrapy.Request(
"https://shopee.ph/api/v4/pdp/get_pc?shop_id=237078553&item_id=6929743700",
cookies=self.shopee_cookies,
headers={"x-api-source":"pc","af-ac-enc-dat":"null"},
callback=self.parse_item,
)
def parse_item(self,response):
open_in_browser(response)
请随意测试,因为我也提供了cookie(因为需要cookie)。现在,正如你所看到的,这段代码实际上在2023年8月初之前就已经工作了。我以前遇到过让它发挥作用的挑战,但多亏了
this answer
我设法获得了产品数据。你甚至可以看到我的
comment there
。这是我在8月初证明它确实有效之前截屏的一张图片。
正如您所看到的,数据就在那里,并且运行良好。多亏了标题
{"x-api-source":"pc","af-ac-enc-dat":"null"}
这使它发挥了作用。然而,截至2023年8月20日,我正在键入此内容。它似乎已经不起作用了。我不知道为什么,但我认为API发生了一些变化。我花了一整天的时间试图弄清楚并打出头球,但没有运气。结果我现在得到的就是这个。
我现在的输出:
{“is_customized”:false,“is_login”:true,“platform”:0,“action_type”:2,“error”:90309999,“tracking_id”:“24d95bd5-40e5-444cd-b30b-885711481170”,“report_extra_info”:“”}
这是实际的产品页面
link
我曾经做过测试。当你做“检查元素”时,你可以在那里看到API->“网络”选项卡。请注意,我现在的输出与我实现之前的输出相同
this solution
。但现在又回来了。所以问题是,有没有办法让它再次发挥作用?我觉得这是我没有做好的事情,但我不知道如何解决,这就是为什么我现在正在寻求帮助,因为我没有解决方案。
