Skip to content

Commit

Permalink
code formatting
Browse files Browse the repository at this point in the history
  • Loading branch information
AndrewKorzh committed Aug 14, 2024
1 parent fa98add commit 9521c09
Show file tree
Hide file tree
Showing 10 changed files with 174 additions and 135 deletions.
2 changes: 1 addition & 1 deletion examples/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,4 @@

PUPPETEER_SERVICE_URL = "http://localhost:3000"

PUPPETEER_LOCAL = False
PUPPETEER_LOCAL = False
34 changes: 13 additions & 21 deletions examples/spiders/form_action.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@
from scrapypuppeteer.actions import Screenshot, FormAction
import base64


class FormActionSpider(scrapy.Spider):
name = 'form_action'
name = "form_action"
start_urls = ["https://www.roboform.com/filling-test-all-fields"]

def start_requests(self):
Expand All @@ -13,34 +14,25 @@ def start_requests(self):

def form_action(self, response):
input_mapping = {
'input[name="02frstname"]': {
"value": "SomeName",
"delay": 50
},
'input[name="05_company"]': {
"value": "SomeCompany",
"delay": 100
},
'input[name="06position"]': {
"value": "SomePosition",
"delay": 100
}
'input[name="02frstname"]': {"value": "SomeName", "delay": 50},
'input[name="05_company"]': {"value": "SomeCompany", "delay": 100},
'input[name="06position"]': {"value": "SomePosition", "delay": 100},
}

yield response.follow(
FormAction(input_mapping),
close_page=False,
callback=self.screenshot
FormAction(input_mapping), close_page=False, callback=self.screenshot
)

def screenshot(self, response):
action = Screenshot(options = {"fullPage": True,})
yield response.follow(action, callback = self.make_screenshot, close_page = False)
action = Screenshot(
options={
"fullPage": True,
}
)
yield response.follow(action, callback=self.make_screenshot, close_page=False)

@staticmethod
def make_screenshot(response: PuppeteerScreenshotResponse, **kwargs):
data = (
response.screenshot
)
data = response.screenshot
with open(f"screenshot.png", "wb") as fh:
fh.write(base64.b64decode(data))
9 changes: 6 additions & 3 deletions examples/spiders/har.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,19 @@


def write_to_file(file_path, content):
with open(file_path, 'a', encoding='utf-8') as file:
with open(file_path, "a", encoding="utf-8") as file:
file.write(content)


class HarSpider(scrapy.Spider):
name = 'har'
name = "har"
start_urls = ["https://github.com/pyppeteer/pyppeteer"]

def start_requests(self):
for url in self.start_urls:
yield PuppeteerRequest(url, callback=self.har, close_page=False, har_recording=True)
yield PuppeteerRequest(
url, callback=self.har, close_page=False, har_recording=True
)

def har(self, response):
yield response.follow(
Expand Down
21 changes: 11 additions & 10 deletions scrapypuppeteer/actions.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,11 @@ class GoTo(PuppeteerServiceAction):
endpoint = "goto"

def __init__(
self, url: str, navigation_options: dict = None, wait_options: dict = None, har_recording: bool = False
self,
url: str,
navigation_options: dict = None,
wait_options: dict = None,
har_recording: bool = False,
):
self.url = url
self.navigation_options = navigation_options
Expand Down Expand Up @@ -223,10 +227,9 @@ def __init__(self, options: dict = None, **kwargs):

def payload(self):
return {"options": self.options}


class Har(PuppeteerServiceAction):


class Har(PuppeteerServiceAction):
"""
The `Har` action is used to capture and retrieve the HTTP Archive (HAR) file,
which contains detailed information about network requests and responses
Expand All @@ -236,14 +239,14 @@ class Har(PuppeteerServiceAction):
you must pass the `har_recording=True` argument to `PuppeteerRequest`
when initiating the request.
"""

endpoint = "har"

def payload(self):
return {}


class FormAction(PuppeteerServiceAction):


class FormAction(PuppeteerServiceAction):
"""
Fill out and submit forms on a webpage.
Expand All @@ -267,11 +270,9 @@ class FormAction(PuppeteerServiceAction):
def __init__(self, input_mapping: dict, submit_button: str = None):
self.input_mapping = input_mapping
self.submit_button = submit_button

def payload(self):
return {"inputMapping": self.input_mapping, "submitButton": self.submit_button}




class RecaptchaSolver(PuppeteerServiceAction):
Expand Down
5 changes: 3 additions & 2 deletions scrapypuppeteer/browser_managers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,16 @@

from abc import ABC, abstractmethod


class BrowserManager(ABC):
@abstractmethod
def process_request(self, request, spider):
pass

@abstractmethod
def close_used_contexts(self):
pass

@abstractmethod
def process_response(self, middleware, request, response, spider):
pass
pass
Loading

0 comments on commit 9521c09

Please sign in to comment.