Skip to content

Commit

Permalink
New: Add worker service to serverless
Browse files Browse the repository at this point in the history
  • Loading branch information
sarvaje committed Sep 12, 2019
1 parent c9f678f commit 739275e
Show file tree
Hide file tree
Showing 10 changed files with 489 additions and 27 deletions.
18 changes: 18 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
.git/
node_modules/
.vscode/
.nyc_output/
coverage/
.editorconfig
.eslintrc.json
.gitignore
.travis.yml
Dockerfile.*
.gitattributes
package-lock.json
function-*/
scripts/
src/functions/azure/
!src/function/azure/workerservice.ts
src/lib/microservices/
!src/lib/microservices/worker-service/
2 changes: 1 addition & 1 deletion .vscode/tasks.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"problemMatcher": "$func-watch",
"isBackground": true,
"dependsOn": "npm: build"
}
},
{
"type": "npm",
"script": "build",
Expand Down
31 changes: 31 additions & 0 deletions Dockerfile.worker
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
FROM mcr.microsoft.com/azure-functions/node:2.0

#Update ubuntu and install dependencies
RUN apt-get update
RUN apt-get upgrade -y
RUN apt-get install -y curl apt-transport-https gnupg vim

# Install nodejs
RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
RUN apt-get install -y nodejs build-essential

# Install chrome
RUN curl -sL https://dl.google.com/linux/linux_signing_key.pub | apt-key add -
RUN echo "deb https://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list
RUN apt-get update
RUN apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst

ENV AzureWebJobsScriptRoot=/home/site/wwwroot \
AzureFunctionsJobHost__Logging__Console__IsEnabled=true

WORKDIR /home/site/wwwroot

COPY package.json /home/site/wwwroot

RUN npm install

# Double check you have the binaries for the Azure Service Bus extension
# before create the image.
COPY . /home/site/wwwroot

RUN npm run build
22 changes: 20 additions & 2 deletions functions.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,22 @@
[
{
"name": "workerservice",
"template": {
"disabled": false,
"bindings": [
{
"type": "serviceBusTrigger",
"direction": "in",
"name": "message",
"queueName": "webhint-jobs",
"connection": "QueueConnection",
"accessRights": "listen"
}
],
"entryPoint": "run",
"scriptFile": "../dist/src/functions/azure/workerservice.js"
}
},
{
"name": "syncservice",
"template": {
Expand All @@ -17,7 +35,7 @@
"scriptFile": "../dist/src/functions/azure/syncservice.js"
}
},
{
{
"name": "statusservice",
"template": {
"disabled": false,
Expand Down Expand Up @@ -83,4 +101,4 @@
"scriptFile": "../dist/src/functions/azure/api-createjob.js"
}
}
]
]
12 changes: 10 additions & 2 deletions host.json
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
{
"version": "2.0"
}
"version": "2.0",
"extensions": {
"serviceBus": {
"messageHandlerOptions": {
"maxAutoRenewDuration": "00:01:00",
"maxConcurrentCalls": 5
}
}
}
}
40 changes: 20 additions & 20 deletions scripts/create-function-templates.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,23 @@ const fs = require('fs');
const rimraf = require('rimraf');

/*
** Azure functions enforce a certain folder structure
** to work/deploy properly. Essentially, they need all
** functions to have a function.json definition file.
** Each of those files have to be within a folder
** (named after the function) on the root of the project.
** This creates the need to have a lot of folders within
** the root of the project making it look very messy.
**
** We've instead created a functions.json file that combines
** all the function definitions into an array of
** (function-name, definition) pair so we can have all of
** them in one file. However, since Azure requires the folder
** structure, this script will split the definitions array
** into individual definitions and place them into
** appropriate folders. Those folders don't get checked into
** source control, and get re-created every time when building.
*/
** Azure functions enforce a certain folder structure
** to work/deploy properly. Essentially, they need all
** functions to have a function.json definition file.
** Each of those files have to be within a folder
** (named after the function) on the root of the project.
** This creates the need to have a lot of folders within
** the root of the project making it look very messy.
**
** We've instead created a functions.json file that combines
** all the function definitions into an array of
** (function-name, definition) pair so we can have all of
** them in one file. However, since Azure requires the folder
** structure, this script will split the definitions array
** into individual definitions and place them into
** appropriate folders. Those folders don't get checked into
** source control, and get re-created every time when building.
*/

const functionsFile = 'functions.json';

Expand All @@ -32,12 +32,12 @@ const main = () => {
json.forEach((node) => {
const name = `function-${node.name}`;

if (fs.existsSync(name)){
if (fs.existsSync(name)){ // eslint-disable-line no-sync
rimraf.sync(name);
}

fs.mkdirSync(name);
fs.writeFileSync(`${name}/function.json`, JSON.stringify(node.template, null, 4), (err) => {
fs.mkdirSync(name); // eslint-disable-line no-sync
fs.writeFileSync(`${name}/function.json`, JSON.stringify(node.template, null, 4), (err) => { // eslint-disable-line no-sync
if (err) {
throw (err);
}
Expand Down
11 changes: 11 additions & 0 deletions src/functions/azure/workerservice.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import { AzureFunction, Context } from '@azure/functions';

import * as workerService from '../../lib/microservices/worker-service/worker-service';
import { IJob } from '../../lib/types';

export const run: AzureFunction = async (context: Context, job: IJob): Promise<void> => {
context.log(`Start scanning: ${job.url} - Part ${job.partInfo ? job.partInfo.part : '-'} of ${job.partInfo ? job.partInfo.totalParts : '-'}`);
await workerService.run(job);
context.log(`Scan end for: ${job.url}`);
context.done();
};
2 changes: 1 addition & 1 deletion src/lib/microservices/scanner-api/scanner-api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { JobStatus, HintStatus } from '../../enums/status';
import { getTime } from '../../common/ntp/ntp';
import { Queue } from '../../common/queue/queue';

const { QueueConnection: queueConnection, DatabaseConnection: dbConnectionString } = process.env; // eslint-disable-line no-process-env
const { QueueConnection: queueConnection } = process.env; // eslint-disable-line no-process-env
let queue: Queue = null;
const moduleName: string = 'Scanner API';
const categories = require('./categories.json');
Expand Down
1 change: 0 additions & 1 deletion src/lib/microservices/sync-service/sync-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import { IssueData } from '../../types/issuedata';

const slowReturnMessage = `webhint didn't return the result fast enough`;
const moduleName: string = 'Sync Function';
const { DatabaseConnection: dbConnectionString } = process.env; // eslint-disable-line no-process-env
const appInsightClient = appInsight.getClient();

/**
Expand Down
Loading

0 comments on commit 739275e

Please sign in to comment.