Scrape release date

.vscode/settings.json

@@ -12,6 +12,12 @@
         "editor.formatOnSave": true
     },
     "cSpell.enabled": true,
-    "cSpell.words": ["Novak", "parens", "vrgamedeals"],
+    "cSpell.words": [
+        "Chrono",
+        "Luxon",
+        "Novak",
+        "parens",
+        "vrgamedeals"
+    ],
     "typescript.preferences.importModuleSpecifierEnding": "js"
 }

package.json

@@ -29,13 +29,16 @@
     },
     "dependencies": {
         "cheerio": "1.0.0-rc.12",
+        "chrono-node": "^2.5.0",
         "express": "4.18.2",
         "express-promise-router": "4.1.1",
+        "luxon": "3.2.1",
         "node-fetch": "3.3.0",
         "pm2": "^5.2.2"
     },
     "devDependencies": {
         "@types/express": "4.17.14",
+        "@types/luxon": "3.2.0",
         "@types/node": "18.11.9",
         "@types/node-fetch": "2.6.2",
         "@typescript-eslint/eslint-plugin": "^5.42.0",

public/scripts/index.js

@@ -199,6 +199,7 @@ function formatAppData(app) {
         countdownTime: 0,
         reviews: '',
         reviewsCount: '',
+        releaseDate: '',
     };
 
     formattedData.type = app.type;
@@ -216,6 +217,7 @@ function formatAppData(app) {
     formattedData.countdownTime = app.countdown.time;
     formattedData.reviews = extractNumberFromPercent(app.reviewsPercent) || app.reviewsPercent;
     formattedData.reviewsCount = app.reviewsCount;
+    formattedData.releaseDate = app.releaseDate;
 
     return formattedData;
 }
@@ -231,8 +233,8 @@ async function retrieveSearchPageData(steamSearchUrl, pageNumber) {
 }
 
 function createMarkdownTable(formattedSearchData) {
-    let header = '| Title | Price (USD) | Discount (%) | Rating (%) | Review Count |';
-    let divider = '| :- | -: | -: | -: | -: |';
+    let header = '| Title | Price (USD) | Discount (%) | Rating (%) | Total Reviews | Release Date |';
+    let divider = '| :- | -: | -: | -: | -: | :- |';
     let result = header + NEW_LINE + divider + NEW_LINE;
 
     for (let app of formattedSearchData) {
@@ -243,7 +245,7 @@ function createMarkdownTable(formattedSearchData) {
 }
 
 function convertToRow(app) {
-    return `| ${app.titleLink} | ${app.price} | ${app.percentOff} | ${app.reviews} | ${app.reviewsCount} |`;
+    return `| ${app.titleLink} | ${app.price} | ${app.percentOff} | ${app.reviews} | ${app.reviewsCount} | ${app.releaseDate} |`;
 }
 
 async function post(url, content, maxAttempts) {

src/models/internal-models.ts

@@ -5,6 +5,7 @@ export interface AppPageData {
     percentOff: string;
     reviewsPercent: string;
     reviewsCount: string;
+    releaseDate: string;
     countdown: CountdownData;
     vrSupport: string;
     link: string;
@@ -24,6 +25,7 @@ export interface GameData {
     percentOff: string;
     reviewsPercent: string;
     reviewsCount: string;
+    releaseDate: string;
 }
 
 export interface GameElementData {

src/services/steam-scraper.ts

@@ -1,5 +1,7 @@
 import * as cheerio from 'cheerio';
 import { Element } from 'cheerio';
+import chrono from 'chrono-node';
+import { DateTime } from 'luxon';
 
 import {
     AppPageData,
@@ -23,11 +25,13 @@ export class SteamScraper {
         let countdown = this.getCountdownFromGameElement(firstGame);
         let vrSupport = this.getVrSupportFromGameElement(firstGame);
         let reviews = this.getReviews(appPageHtml);
+        let releaseDate = this.getReleaseDate(appPageHtml);
 
         return {
             title,
             ...gameData,
             ...reviews,
+            releaseDate,
             countdown,
             vrSupport,
             link: undefined,
@@ -92,6 +96,29 @@ export class SteamScraper {
         return title;
     }
 
+    private getReleaseDate(appPageHtml: string): string {
+        let $ = cheerio.load(appPageHtml);
+
+        let releaseDate = '';
+
+        let gameDetailsElement = $(
+            '.game_details .details_block:contains("Release Date:")'
+        ).first();
+
+        if (gameDetailsElement) {
+            let gameDetailsContent = gameDetailsElement.html().trim();
+            let releaseDateText = RegexUtils.extractReleaseDate(gameDetailsContent);
+            if (releaseDateText) {
+                let releaseJsDate = chrono.parseDate(releaseDateText);
+                if (releaseJsDate) {
+                    releaseDate = DateTime.fromJSDate(releaseJsDate).toFormat('yyyy-MM-dd');
+                }
+            }
+        }
+
+        return releaseDate;
+    }
+
     private getReviews(appPageHtml: string): ReviewData {
         let $ = cheerio.load(appPageHtml);
 
@@ -124,6 +151,7 @@ export class SteamScraper {
             percentOff: '',
             reviewsPercent: '',
             reviewsCount: '',
+            releaseDate: '',
         };
 
         let title = $('div.search_name > span.title').text().trim();
@@ -170,6 +198,14 @@ export class SteamScraper {
             }
         }
 
+        let releaseDateText = $('div.search_released').text().trim();
+        if (releaseDateText) {
+            let releaseJsDate = chrono.parseDate(releaseDateText);
+            if (releaseJsDate) {
+                gameData.releaseDate = DateTime.fromJSDate(releaseJsDate).toFormat('yyyy-MM-dd');
+            }
+        }
+
         return gameData;
     }
 

src/utils/regex-utils.ts

@@ -1,4 +1,5 @@
 const TITLE_REGEX = /<b>Title:<\/b>(.*)<br>/;
+const RELEASE_DATE_REGEX = /<b>Release Date:<\/b>(.*)<br>/;
 const PERCENT_REGEX = /(\d+%)/;
 const REVIEWS_COUNT_REGEX = /([\d,]+) user review/;
 const DISCOUNT_COUNTDOWN_REGEX = /DiscountCountdown,[ ]*([\d]{7,})/;
@@ -11,6 +12,13 @@ export abstract class RegexUtils {
         }
     }
 
+    public static extractReleaseDate(input: string): string {
+        let match = RELEASE_DATE_REGEX.exec(input);
+        if (match) {
+            return match[1].trim();
+        }
+    }
+
     public static extractPercent(input: string): string {
         let match = PERCENT_REGEX.exec(input);
         if (match) {