Skip to content

Commit

Permalink
Use mock server for selenium tests, fix #1116 (#1119)
Browse files Browse the repository at this point in the history 
* Use mock server for selenium tests, fix #1116

Signed-off-by: Julien Nioche <[email protected]>

* Use Jetty and set a local hostname for the website to visit

Signed-off-by: Richard Zowalla <[email protected]>

* Removed unused test dependency + fixed typo

Signed-off-by: Julien Nioche <[email protected]>

---------

Signed-off-by: Julien Nioche <[email protected]>
Signed-off-by: Richard Zowalla <[email protected]>
Co-authored-by: Richard Zowalla <[email protected]>
  • Loading branch information
@jnioche @rzo1
jnioche and rzo1 authored Nov 8, 2023
1 parent ef31e50 commit 1556212
Show file tree
Hide file tree
Showing 2 changed files with 99 additions and 19 deletions.
11 changes: 10 additions & 1 deletion core/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
<?xml version="1.0"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
Expand Down Expand Up @@ -131,6 +133,13 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
<version>3.0.0</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
Expand Down
107 changes: 89 additions & 18 deletions core/src/test/java/com/digitalpebble/stormcrawler/protocol/selenium/ProtocolTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,16 @@
*/
package com.digitalpebble.stormcrawler.protocol.selenium;

import static org.awaitility.Awaitility.await;
import static org.junit.Assert.assertNotEquals;

import com.digitalpebble.stormcrawler.Metadata;
import com.digitalpebble.stormcrawler.protocol.ProtocolResponse;
import jakarta.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.OutputStream;
import java.net.ServerSocket;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
Expand All @@ -24,7 +32,14 @@
import org.apache.commons.lang.mutable.MutableBoolean;
import org.apache.storm.Config;
import org.apache.storm.utils.MutableObject;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.handler.AbstractHandler;
import org.eclipse.jetty.server.handler.HandlerList;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
Expand All @@ -45,14 +60,47 @@ public class ProtocolTest {

private static final Logger LOG = LoggerFactory.getLogger(ProtocolTest.class);

private static final DockerImageName IMAGE =
DockerImageName.parse("selenium/standalone-chrome:118.0");
private static final DockerImageName SELENIUM_IMAGE =
DockerImageName.parse("selenium/standalone-chrome");

@Rule
public BrowserWebDriverContainer<?> chrome =
new BrowserWebDriverContainer<>(IMAGE)
new BrowserWebDriverContainer<>(SELENIUM_IMAGE)
.withCapabilities(new ChromeOptions())
.withRecordingMode(VncRecordingMode.SKIP, null);
.withRecordingMode(VncRecordingMode.SKIP, null)
.withAccessToHost(true)
.withExtraHost("website.test", "host-gateway");

private static Server httpServer;
private static final Integer HTTP_PORT = findRandomOpenPortOnAllLocalInterfaces();

@BeforeClass
public static void initJetty() throws Exception {
assertNotEquals(Integer.valueOf(-1), HTTP_PORT);
httpServer = new Server(HTTP_PORT);

final HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[] {new WildcardResourceHandler()});
httpServer.setHandler(handlers);
httpServer.start();
}

@AfterClass
public static void stopJetty() {
try {
httpServer.stop();
} catch (Exception ignored) {

}
}

private static Integer findRandomOpenPortOnAllLocalInterfaces() {
try (ServerSocket socket = new ServerSocket(0)) {
return socket.getLocalPort();
} catch (IOException e) {
return -1;
}
}

public RemoteDriverProtocol getProtocol() {

Expand Down Expand Up @@ -93,30 +141,30 @@ public RemoteDriverProtocol getProtocol() {
return protocol;
}

@Test
/**
* you can configure one instance of Selenium to talk to multiple drivers but can't have a
* multiple instances of the protocol. If there is only one instance and one target, you must
* wait...
*
* @throws InterruptedException
*/
@Test
public void testBlocking() {

// TODO find a way of not hitting a real URL
public void testBlocking() throws InterruptedException {
RemoteDriverProtocol protocol = getProtocol();

MutableBoolean noException = new MutableBoolean(true);

MutableObject endTimeFirst = new MutableObject();
MutableObject startTimeSecond = new MutableObject();

await().until(() -> httpServer.isRunning());

final String url = "http://website.test" + ":" + HTTP_PORT + "/";

new Thread(
() -> {
try {
ProtocolResponse response =
protocol.getProtocolOutput(
"https://stormcrawler.net/", new Metadata());
protocol.getProtocolOutput(url, new Metadata());
endTimeFirst.setObject(
Instant.parse(
response.getMetadata()
Expand All @@ -132,8 +180,7 @@ public void testBlocking() throws InterruptedException {
() -> {
try {
ProtocolResponse response =
protocol.getProtocolOutput(
"https://stormcrawler.net/", new Metadata());
protocol.getProtocolOutput(url, new Metadata());
startTimeSecond.setObject(
Instant.parse(
response.getMetadata()
Expand All @@ -145,17 +192,41 @@ public void testBlocking() throws InterruptedException {
})
.start();

while (endTimeFirst.getObject() == null || startTimeSecond.getObject() == null) {
Thread.sleep(10);
}
await().until(
() ->
endTimeFirst.getObject() != null
&& startTimeSecond.getObject() != null);

Instant etf = (Instant) endTimeFirst.getObject();
Instant sts = (Instant) startTimeSecond.getObject();

// check that the second call started AFTER the first one finished
Assert.assertEquals(true, etf.isBefore(sts));
Assert.assertTrue(etf.isBefore(sts));

Assert.assertEquals(true, noException.booleanValue());
Assert.assertTrue(noException.booleanValue());
protocol.cleanup();
}

public static class WildcardResourceHandler extends AbstractHandler {

@Override
public void handle(
String s,
Request baseRequest,
jakarta.servlet.http.HttpServletRequest httpServletRequest,
jakarta.servlet.http.HttpServletResponse response)
throws IOException {
if (response.isCommitted() || baseRequest.isHandled()) return;

baseRequest.setHandled(true);

final String content = "Success!";
response.setStatus(HttpServletResponse.SC_OK);
response.setContentType("text/html");
response.setContentLength(content.length());
try (OutputStream out = response.getOutputStream()) {
out.write(content.getBytes(StandardCharsets.UTF_8));
}
}
}
}

0 comments on commit 1556212

Please sign in to comment.