diff --git a/.gitignore b/.gitignore index 2576922..f5ee73d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,9 +6,6 @@ yarn-debug.log* yarn-error.log* lerna-debug.log* -# Diagnostic reports (https://nodejs.org/api/report.html) -report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json - # Runtime data pids *.pid @@ -124,3 +121,6 @@ dist npm-debug.log* yarn-debug.log* yarn-error.log* + +# Redis +worker/data diff --git a/client/yarn.lock b/client/yarn.lock index 905b93d..8a6827b 100644 --- a/client/yarn.lock +++ b/client/yarn.lock @@ -9204,10 +9204,10 @@ serialize-javascript@^1.7.0: resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-1.9.1.tgz#cfc200aef77b600c47da9bb8149c943e798c2fdb" integrity sha512-0Vb/54WJ6k5v8sSWN09S0ora+Hnr+cX40r9F170nT+mSkaxltoE/7R3OrIdBSUv1OoiobH1QoWQbCnAO+e8J1A== -serialize-javascript@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-2.1.0.tgz#9310276819efd0eb128258bb341957f6eb2fc570" - integrity sha512-a/mxFfU00QT88umAJQsNWOnUKckhNCqOl028N48e7wFmo2/EHpTo9Wso+iJJCMrQnmFvcjto5RJdAHEvVhcyUQ== +serialize-javascript@^2.1.1: + version "2.1.2" + resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-2.1.2.tgz#ecec53b0e0317bdc95ef76ab7074b7384785fa61" + integrity sha512-rs9OggEUF0V4jUSecXazOYsLfu7OGK2qIn3c7IPBiffz32XniEp/TX9Xmc9LQfK2nQ2QKHvZ2oygKUGU0lG4jQ== serve-index@^1.9.1: version "1.9.1" diff --git a/worker/index.js b/worker/index.js new file mode 100644 index 0000000..f0f5a73 --- /dev/null +++ b/worker/index.js @@ -0,0 +1,5 @@ +const CronJob = require('cron').CronJob; + +const fetchGithub = require('./tasks/fetch-github'); + +new CronJob('* * * * *', fetchGithub, null, true, 'Asia/Kathmandu'); diff --git a/worker/makefile b/worker/makefile new file mode 100644 index 0000000..2e713bd --- /dev/null +++ b/worker/makefile @@ -0,0 +1,16 @@ +download-redis: + docker pull redis + +setup-volume: + mkdir data + +start-redis: + docker run --name redis-job-board \ + -p 6379:6379 \ + --restart unless-stopped \ + -v ${PWD}/data:/data\ + -d redis redis-server --appendonly yes + +stop-redis: + docker stop redis-job-board + docker rm redis-job-board \ No newline at end of file diff --git a/worker/package.json b/worker/package.json new file mode 100644 index 0000000..252a273 --- /dev/null +++ b/worker/package.json @@ -0,0 +1,13 @@ +{ + "name": "worker", + "version": "0.1.0", + "description": "Cron worker to fetch data", + "main": "index.js", + "author": "Ajan Lal Shrestha", + "license": "MIT", + "dependencies": { + "cron": "1.7.2", + "node-fetch": "2.6.0", + "redis": "2.8.0" + } +} diff --git a/worker/tasks/fetch-github.js b/worker/tasks/fetch-github.js new file mode 100644 index 0000000..f57576b --- /dev/null +++ b/worker/tasks/fetch-github.js @@ -0,0 +1,32 @@ +const fetch = require('node-fetch'); +const redis = require('redis'), + client = redis.createClient(); +const {promisify} = require('util'); +const getAsync = promisify(client.get).bind(client); +const setAsync = promisify(client.set).bind(client); + +const baseURL = 'https://jobs.github.com/positions.json'; + +const fetchGithub = async () => { + let resultCount = 1, + onPage = 0; + const allJobs = []; + + while (resultCount > 0) { + const res = await fetch(`${baseURL}?page=${onPage}`); + const jobs = await res.json(); + allJobs.push(...jobs); + resultCount = jobs.length; + console.log(`got ${resultCount} jobs`); + onPage++; + } + + console.log(`got ${allJobs.length} jobs`); + const successs = await setAsync('github', JSON.stringify(allJobs)); + + console.log(successs); +}; + +fetchGithub(); + +module.exports = fetchGithub; diff --git a/worker/yarn.lock b/worker/yarn.lock new file mode 100644 index 0000000..897387f --- /dev/null +++ b/worker/yarn.lock @@ -0,0 +1,51 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +cron@1.7.2: + version "1.7.2" + resolved "https://registry.yarnpkg.com/cron/-/cron-1.7.2.tgz#2ea1f35c138a07edac2ac5af5084ed6fee5723db" + integrity sha512-+SaJ2OfeRvfQqwXQ2kgr0Y5pzBR/lijf5OpnnaruwWnmI799JfWr2jN2ItOV9s3A/+TFOt6mxvKzQq5F0Jp6VQ== + dependencies: + moment-timezone "^0.5.x" + +double-ended-queue@^2.1.0-0: + version "2.1.0-0" + resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c" + integrity sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw= + +moment-timezone@^0.5.x: + version "0.5.27" + resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.27.tgz#73adec8139b6fe30452e78f210f27b1f346b8877" + integrity sha512-EIKQs7h5sAsjhPCqN6ggx6cEbs94GK050254TIJySD1bzoM5JTYDwAU1IoVOeTOL6Gm27kYJ51/uuvq1kIlrbw== + dependencies: + moment ">= 2.9.0" + +"moment@>= 2.9.0": + version "2.24.0" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b" + integrity sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg== + +node-fetch@2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" + integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== + +redis-commands@^1.2.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.5.0.tgz#80d2e20698fe688f227127ff9e5164a7dd17e785" + integrity sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg== + +redis-parser@^2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-2.6.0.tgz#52ed09dacac108f1a631c07e9b69941e7a19504b" + integrity sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs= + +redis@2.8.0: + version "2.8.0" + resolved "https://registry.yarnpkg.com/redis/-/redis-2.8.0.tgz#202288e3f58c49f6079d97af7a10e1303ae14b02" + integrity sha512-M1OkonEQwtRmZv4tEWF2VgpG0JWJ8Fv1PhlgT5+B+uNq2cA3Rt1Yt/ryoR+vQNOQcIEgdCdfH0jr3bDpihAw1A== + dependencies: + double-ended-queue "^2.1.0-0" + redis-commands "^1.2.0" + redis-parser "^2.6.0"