From 1f5805568250b4c18c6da034e2a26eb27a8a2c21 Mon Sep 17 00:00:00 2001 From: matthew Date: Mon, 1 Jul 2024 17:11:15 +0300 Subject: [PATCH] Errback and CloseContextRequest fix --- scrapypuppeteer/middleware.py | 5 +++++ scrapypuppeteer/request.py | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scrapypuppeteer/middleware.py b/scrapypuppeteer/middleware.py index 9d1ffc0..8b8919d 100644 --- a/scrapypuppeteer/middleware.py +++ b/scrapypuppeteer/middleware.py @@ -249,6 +249,11 @@ def close_used_contexts(self, spider): msg=f"Successfully closed {len(request.contexts)} contexts with request {response.request}", ) ) + dfd.addErrback( + lambda _: self.service_logger.log( + level=logging.WARNING, msg="Could not close contexts" + ) + ) raise DontCloseSpider() diff --git a/scrapypuppeteer/request.py b/scrapypuppeteer/request.py index 24e968e..a3d55a7 100644 --- a/scrapypuppeteer/request.py +++ b/scrapypuppeteer/request.py @@ -119,9 +119,9 @@ def __init__(self, contexts: List, **kwargs): self.is_valid_url = True url = kwargs.pop("url", "://") # Incorrect url. To be replaced in middleware - kwargs["method"] = "POST", - kwargs["headers"] = Headers({"Content-Type": "application/json"}), - kwargs["body"] = json.dumps(self.contexts), + kwargs["method"] = "POST" + kwargs["headers"] = Headers({"Content-Type": "application/json"}) + kwargs["body"] = json.dumps(self.contexts) super().__init__(url, **kwargs)