Bugfix nofollow instructions in rel tags ignored when the tag has mor…

…e than one value (#1362) Signed-off-by: Julien Nioche <[email protected]>
apache · Oct 21, 2024 · 2fe8d92 · 2fe8d92
1 parent 6a1fafa
commit 2fe8d92
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 1 deletion.
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
@@ -30,6 +30,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.stream.Stream;
 import org.apache.commons.lang.StringUtils;
 import org.apache.storm.metric.api.MultiCountMetric;
 import org.apache.storm.task.OutputCollector;
@@ -269,7 +270,10 @@ public void execute(Tuple tuple) {
                 final URL baseURL = new URL(url);
                 for (Element link : links) {
                     // nofollow
-                    boolean noFollow = "nofollow".equalsIgnoreCase(link.attr("rel"));
+                    String[] relkeywords = link.attr("rel").split(" ");
+                    boolean noFollow =
+                            Stream.of(relkeywords).anyMatch(x -> x.equalsIgnoreCase("nofollow"));
+
                     // remove altogether
                     if (noFollow && robots_noFollow_strict) {
                         continue;

diff --git a/core/src/test/resources/digitalpebble.com.html b/core/src/test/resources/digitalpebble.com.html
@@ -50,6 +50,7 @@
 </script>
 
 <a rel="nofollow" href="inexistent.html"/>
+<a rel="nofollow somevalue" href="another_inexistent.html"/>
 
 <body style="color: rgb(0, 0, 0); background-color: rgb(255, 255, 255);"
 alink="#000000" link="#000000" vlink="#000000">