Replace tripadvisor sitemap example with stormcrawler.apache.org content

Replace guardian.rss feed with stormcrawler.apache.org
apache · Nov 22, 2024 · 0ae250f · 0ae250f
1 parent 60e195d
commit 0ae250f
Show file tree

Hide file tree

Showing 7 changed files with 89 additions and 3,659 deletions.
diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/FeedParserBoltTest.java
@@ -40,7 +40,7 @@ void setupParserBolt() {
     }
 
     private void checkOutput() {
-        Assertions.assertEquals(170, output.getEmitted(Constants.StatusStreamName).size());
+        Assertions.assertEquals(7, output.getEmitted(Constants.StatusStreamName).size());
         List<Object> fields = output.getEmitted(Constants.StatusStreamName).get(0);
         Assertions.assertEquals(3, fields.size());
     }
@@ -51,7 +51,7 @@ void testFeedParsing() throws IOException {
         Metadata metadata = new Metadata();
         // specify that it is a Feed file
         metadata.setValue(FeedParserBolt.isFeedKey, "true");
-        parse("http://www.guardian.com/Feed.xml", "guardian.rss", metadata);
+        parse("https://stormcrawler.apache.org/rss.xml", "stormcrawler.rss", metadata);
         checkOutput();
     }
 
@@ -66,7 +66,7 @@ void testFeedParsingNoMT() throws IOException {
         Metadata metadata = new Metadata();
         // set mime-type
         metadata.setValue("http." + HttpHeaders.CONTENT_TYPE, "application/rss+xml");
-        parse("http://www.guardian.com/feed.xml", "guardian.rss", metadata);
+        parse("https://stormcrawler.apache.org/rss.xml", "stormcrawler.rss", metadata);
         checkOutput();
     }
 
@@ -78,7 +78,7 @@ void testFeedParsingDetextBytes() throws IOException {
         bolt.prepare(
                 parserConfig, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
         Metadata metadata = new Metadata();
-        parse("http://www.guardian.com/feed.xml", "guardian.rss", metadata);
+        parse("https://stormcrawler.apache.org/rss.xml", "stormcrawler.rss", metadata);
         checkOutput();
     }
 

diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/SiteMapParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/SiteMapParserBoltTest.java
@@ -68,8 +68,8 @@ void testSitemapIndexParsing() throws IOException {
         // and its mime-type
         metadata.setValue(HttpHeaders.CONTENT_TYPE, "application/xml");
         parse(
-                "http://www.tripadvisor.com/sitemap-index.xml",
-                "tripadvisor.sitemap.index.xml",
+                "http://stormcrawler.apache.org/sitemap-index.xml",
+                "stormcrawler.sitemap.index.xml",
                 metadata);
         for (List<Object> fields : output.getEmitted(Constants.StatusStreamName)) {
             Metadata parsedMetadata = (Metadata) fields.get(1);
@@ -85,8 +85,11 @@ void testGzipSitemapParsing() throws IOException {
         Metadata metadata = new Metadata();
         // specify that it is a sitemap file
         metadata.setValue(SiteMapParserBolt.isSitemapKey, "true");
-        parse("https://www.tripadvisor.com/sitemap.xml.gz", "tripadvisor.sitemap.xml.gz", metadata);
-        Assertions.assertEquals(50001, output.getEmitted(Constants.StatusStreamName).size());
+        parse(
+                "https://stormcrawler.apache.org/sitemap.xml.gz",
+                "stormcrawler.sitemap.xml.gz",
+                metadata);
+        Assertions.assertEquals(7, output.getEmitted(Constants.StatusStreamName).size());
     }
 
     @Test

diff --git a/core/src/test/resources/guardian.rss b/core/src/test/resources/guardian.rss
diff --git a/core/src/test/resources/stormcrawler.rss b/core/src/test/resources/stormcrawler.rss
@@ -0,0 +1,58 @@
+<rss version="2.0">
+  <channel>
+    <title>StormCrawler Website Updates</title>
+    <link>https://stormcrawler.apache.org/</link>
+    <description>Latest updates and information from the StormCrawler website.</description>
+    <language>en-us</language>
+    <lastBuildDate>Sat, 19 Oct 2024 11:21:53 +0000</lastBuildDate>
+    <pubDate>Sat, 19 Oct 2024 11:21:53 +0000</pubDate>
+
+    <item>
+      <title>StormCrawler Homepage</title>
+      <link>https://stormcrawler.apache.org/</link>
+      <description>Main page of the StormCrawler website.</description>
+      <pubDate>Sat, 19 Oct 2024 11:21:53 +0000</pubDate>
+      <guid>https://stormcrawler.apache.org/</guid>
+    </item>
+
+    <item>
+      <title>StormCrawler Index Page</title>
+      <link>https://stormcrawler.apache.org/index.html</link>
+      <description>Index page of the StormCrawler website.</description>
+      <pubDate>Sat, 19 Oct 2024 11:21:53 +0000</pubDate>
+      <guid>https://stormcrawler.apache.org/index.html</guid>
+    </item>
+
+    <item>
+      <title>StormCrawler Download</title>
+      <link>https://stormcrawler.apache.org/download/index.html</link>
+      <description>Download page for StormCrawler resources.</description>
+      <pubDate>Sat, 19 Oct 2024 11:21:53 +0000</pubDate>
+      <guid>https://stormcrawler.apache.org/download/index.html</guid>
+    </item>
+
+    <item>
+      <title>Getting Started with StormCrawler</title>
+      <link>https://stormcrawler.apache.org/getting-started/</link>
+      <description>Guide to getting started with StormCrawler.</description>
+      <pubDate>Sat, 19 Oct 2024 11:21:53 +0000</pubDate>
+      <guid>https://stormcrawler.apache.org/getting-started/</guid>
+    </item>
+
+    <item>
+      <title>StormCrawler FAQ</title>
+      <link>https://stormcrawler.apache.org/faq/</link>
+      <description>Frequently asked questions about StormCrawler.</description>
+      <pubDate>Sat, 19 Oct 2024 11:21:53 +0000</pubDate>
+      <guid>https://stormcrawler.apache.org/faq/</guid>
+    </item>
+
+    <item>
+      <title>StormCrawler Support</title>
+      <link>https://stormcrawler.apache.org/support/</link>
+      <description>Support information for StormCrawler users.</description>
+      <pubDate>Sat, 19 Oct 2024 11:21:53 +0000</pubDate>
+      <guid>https://stormcrawler.apache.org/support/</guid>
+    </item>
+  </channel>
+</rss>
diff --git a/...t/resources/tripadvisor.sitemap.index.xml → .../resources/stormcrawler.sitemap.index.xml b/...t/resources/tripadvisor.sitemap.index.xml → .../resources/stormcrawler.sitemap.index.xml
@@ -18,24 +18,24 @@ specific language governing permissions and limitations
 under the License.
 -->
 <sitemapindex
-  xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9
+        xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9
                       http://www.sitemaps.org/schemas/sitemap/0.9/siteindex.xsd">
-<sitemap>
-  <loc>https://www.tripadvisor.com/sitemap/2/en_US/sitemap-1806509-en_US-hotel_review-1686849999.xml.gz</loc>
-  <lastmod>2023-06-15T17:26:39Z</lastmod>
-</sitemap>
-<sitemap>
-  <loc>https://www.tripadvisor.com/sitemap/2/en_US/sitemap-1806530-en_US-hotel_review-1686850054.xml.gz</loc>
-  <lastmod>2023-06-15T17:27:34Z</lastmod>
-</sitemap>
-<sitemap>
-  <loc>https://www.tripadvisor.com/sitemap/2/en_US/sitemap-1806537-en_US-hotel_review-1686850072.xml.gz</loc>
-  <lastmod>2023-06-15T17:27:52Z</lastmod>
-</sitemap>
-<sitemap>
-  <loc>https://www.tripadvisor.com/sitemap/2/en_US/sitemap-1841024-en_US-hotel_review-1694976638.xml.gz</loc>
-  <lastmod>2023-09-17T18:50:38Z</lastmod>
-</sitemap>
-</sitemapindex>
+  <sitemap>
+    <loc>https://stormcrawler.apache.org/sitemap-001.xml.gz</loc>
+    <lastmod>2024-10-19T11:21:53Z</lastmod>
+  </sitemap>
+  <sitemap>
+    <loc>https://stormcrawler.apache.org/sitemap-002.xml.gz</loc>
+    <lastmod>2024-10-19T11:21:53Z</lastmod>
+  </sitemap>
+  <sitemap>
+    <loc>https://stormcrawler.apache.org/sitemap-003.xml.gz</loc>
+    <lastmod>2024-10-19T11:21:53Z</lastmod>
+  </sitemap>
+  <sitemap>
+    <loc>https://stormcrawler.apache.org/sitemap-004.xml.gz</loc>
+    <lastmod>2024-10-19T11:21:53Z</lastmod>
+  </sitemap>
+</sitemapindex>
diff --git a/core/src/test/resources/stormcrawler.sitemap.xml.gz b/core/src/test/resources/stormcrawler.sitemap.xml.gz
diff --git a/core/src/test/resources/tripadvisor.sitemap.xml.gz b/core/src/test/resources/tripadvisor.sitemap.xml.gz