Skip to content

Commit

Permalink
add examples
Browse files Browse the repository at this point in the history
  • Loading branch information
AndrewKorzh committed Aug 12, 2024
1 parent cc97fec commit fa98add
Show file tree
Hide file tree
Showing 3 changed files with 74 additions and 0 deletions.
2 changes: 2 additions & 0 deletions examples/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,5 @@
}

PUPPETEER_SERVICE_URL = "http://localhost:3000"

PUPPETEER_LOCAL = False
46 changes: 46 additions & 0 deletions examples/spiders/form_action.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import scrapy
from scrapypuppeteer import PuppeteerRequest, PuppeteerScreenshotResponse
from scrapypuppeteer.actions import Screenshot, FormAction
import base64

class FormActionSpider(scrapy.Spider):
name = 'form_action'
start_urls = ["https://www.roboform.com/filling-test-all-fields"]

def start_requests(self):
for url in self.start_urls:
yield PuppeteerRequest(url, callback=self.form_action, close_page=False)

def form_action(self, response):
input_mapping = {
'input[name="02frstname"]': {
"value": "SomeName",
"delay": 50
},
'input[name="05_company"]': {
"value": "SomeCompany",
"delay": 100
},
'input[name="06position"]': {
"value": "SomePosition",
"delay": 100
}
}

yield response.follow(
FormAction(input_mapping),
close_page=False,
callback=self.screenshot
)

def screenshot(self, response):
action = Screenshot(options = {"fullPage": True,})
yield response.follow(action, callback = self.make_screenshot, close_page = False)

@staticmethod
def make_screenshot(response: PuppeteerScreenshotResponse, **kwargs):
data = (
response.screenshot
)
with open(f"screenshot.png", "wb") as fh:
fh.write(base64.b64decode(data))
26 changes: 26 additions & 0 deletions examples/spiders/har.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import scrapy
from scrapypuppeteer import PuppeteerRequest
from scrapypuppeteer.actions import Har


def write_to_file(file_path, content):
with open(file_path, 'a', encoding='utf-8') as file:
file.write(content)

class HarSpider(scrapy.Spider):
name = 'har'
start_urls = ["https://github.com/pyppeteer/pyppeteer"]

def start_requests(self):
for url in self.start_urls:
yield PuppeteerRequest(url, callback=self.har, close_page=False, har_recording=True)

def har(self, response):
yield response.follow(
Har(),
close_page=False,
callback=self.save_har,
)

def save_har(self, response):
write_to_file("result.har", response.har)

0 comments on commit fa98add

Please sign in to comment.