Add spidergram config - TODO make the aragnodb link dynamic

rpkoller · Jun 11, 2023 · 91de8a8 · 91de8a8
1 parent 7316349
commit 91de8a8
Showing 1 changed file with 24 additions and 0 deletions.
diff --git a/spidergram.config.json b/spidergram.config.json
@@ -0,0 +1,24 @@
+{
+  "arango": {
+    "databaseName": "db",
+    "url": "http://grammer.ddev.site:8529",
+    "auth": {
+      "username": "root",
+      "password": "db"
+    }
+  },
+  "spider": {
+    "urlOptions": {
+      "regions": {
+        "header": "header",
+        "footer": "footer",
+        "main": "body"
+      }
+    },
+    "downloadMimeTypes": ["application/pdf"],
+    "userAgent": "MyCustomSpider"
+  },
+  "urlNormalizer": {
+    "discardTrailingSlash": true
+  }
+}