I try to use SplashRequest with endpoint='execute' with below code to send a post request,but the result shows I didn't succeed.
import re
import sys
import os
import scrapy
from scrapy_splash import SplashRequest
from crawler.items import CrawlerItem
class Exp10itSpider(scrapy.Spider):
name = "test"
lua_script = """
function main(splash, args)
assert(splash:go(args.url))
assert(splash:wait(0.5))
return splash:html()
end
"""
def start_requests(self):
urls = [
'https://httpbin.org/post^sss=lalala'
]
for url in urls:
if "^" in url:
post_url_list = url.split("^")
post_url = post_url_list[0]
post_data = post_url_list[1]
yield SplashRequest(post_url, self.parse, endpoint='execute', magic_response=True, meta={'handle_httpstatus_all': True}, args={'lua_source': self.lua_script, 'http_method': 'POST', 'body': post_data})
def parse(self, response):
input("body is:")
print(response.body)
However,I can not post it successully,the return content shows me I don't post it well.Can you help me on post request with SplashRequest in scrapy_splash?