Errback and CloseContextRequest fix

ispras · Jul 1, 2024 · 1f58055 · 1f58055
1 parent d1726b4
commit 1f58055
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/scrapypuppeteer/middleware.py b/scrapypuppeteer/middleware.py
@@ -249,6 +249,11 @@ def close_used_contexts(self, spider):
                     msg=f"Successfully closed {len(request.contexts)} contexts with request {response.request}",
                 )
             )
+            dfd.addErrback(
+                lambda _: self.service_logger.log(
+                    level=logging.WARNING, msg="Could not close contexts"
+                )
+            )
             raise DontCloseSpider()
 
 

diff --git a/scrapypuppeteer/request.py b/scrapypuppeteer/request.py
@@ -119,9 +119,9 @@ def __init__(self, contexts: List, **kwargs):
             self.is_valid_url = True
         url = kwargs.pop("url", "://")  # Incorrect url. To be replaced in middleware
 
-        kwargs["method"] = "POST",
-        kwargs["headers"] = Headers({"Content-Type": "application/json"}),
-        kwargs["body"] = json.dumps(self.contexts),
+        kwargs["method"] = "POST"
+        kwargs["headers"] = Headers({"Content-Type": "application/json"})
+        kwargs["body"] = json.dumps(self.contexts)
 
         super().__init__(url, **kwargs)