Containerized the Interface and Visualization And Setup Github Actions

.github/workflows/interface_test.yml

@@ -0,0 +1,36 @@
+name: YABA interface tests
+
+on:
+  push:
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  interface_test:
+
+    runs-on: ubuntu-latest
+    defaults:
+        run:
+          working-directory: interface
+
+    strategy:
+      matrix:
+        node-version: [12.x, 14.x]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Set up Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    - name: Install dependencies
+      run: npm install
+
+    - name: Run the tests and generate coverage report
+      run: npm test -- --coverage
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1

.github/workflows/linter.yml

@@ -0,0 +1,25 @@
+name: YABA Code Linter
+
+on:
+  push:
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  lint:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      - name: Lint Code Base
+        uses: docker://github/super-linter:v3
+        env:
+          PYTHON_PYLINT_CONFIG_FILE: .pylintrc
+          VALIDATE_ALL_CODEBASE: false
+          VALIDATE_PYTHON_PYLINT: true
+          VALIDATE_JAVASCRIPT_ES: true
+          DEFAULT_BRANCH: master
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/py_tests.yml

@@ -0,0 +1,54 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: YABA app and client tests
+
+on:
+  push:
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  py_test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        code: [ app, client]
+        include:
+         - code: app
+           path: ./yaba_test/test_yaba.py
+         - code: client
+           path: ./yaba_client_test/test_yaba_client.py
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Install dependencies
+        run: |
+              pip install --upgrade pip
+              pip install pytest
+              pip install pytest-cov
+              pip install docker-compose
+
+      - name: Set-up the project
+        run: |
+              docker-compose up -d postgres
+              docker-compose run --rm bety initialize
+              docker-compose up -d
+              docker ps -a
+              docker logs betydb-yaba_yaba_api_1
+
+      - name: Test with pytest and generate coverage report
+        run: |
+              sleep 20
+              pytest --cov= ${{ matrix.path }} --cov-report=xml
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1

.gitignore

@@ -104,4 +104,4 @@ venv.bak/
 .mypy_cache/
 
 # node modules
-node_modules/
+node_modules/

app/Meta.py

@@ -7,9 +7,7 @@
 import traceback
 import logging
 import pandas as pd
-
-from flask import make_response,Response,jsonify
-from flask import json
+from flask import make_response,jsonify
 from db import *
 from werkzeug import secure_filename,FileStorage
 from sqlalchemy.exc import IntegrityError
@@ -329,7 +327,7 @@ def insert_citations(username,fileName):
             insert_table(table='citations',data=data)
             msg = {'Message' : 'Successfully inserted',
                    'Table Affected' : 'Citations',
-                    'Lines Inserted': data.shape[0]}
+                   'Lines Inserted': data.shape[0]}
 
             return make_response(jsonify(msg), 201)
 
@@ -379,7 +377,7 @@ def insert_experimentSites(fileName):
             insert_table(table='experiments_sites',data=new_data)
             msg = {'Message' : 'Successfully inserted',
                    'Table Affected' : 'Experiments_sites',
-                    'Lines Inserted': data.shape[0]}
+                   'Lines Inserted': data.shape[0]}
 
             return make_response(jsonify(msg), 201)
 
@@ -431,7 +429,7 @@ def insert_experimentTreatments(fileName):
             insert_table(table='experiments_treatments',data=new_data)
             msg = {'Message' : 'Successfully inserted',
                    'Table Affected' : 'Experiments_treatments',
-                    'Lines Inserted': data.shape[0]}
+                   'Lines Inserted': data.shape[0]}
 
             return make_response(jsonify(msg), 201)
 
@@ -535,14 +533,12 @@ def insert_citationsSites(fileName):
             insert_table(table='citations_sites',data=new_data)
             msg = {'Message' : 'Successfully inserted',
                    'Table Affected' : 'Citations_sites',
-                   'col' : columns,
                    'Lines Inserted': data.shape[0]}
 
             return make_response(jsonify(msg), 201)
 
         else:
             msg = {'Message' : 'File not acceptable and Check the format of file or columns',
-                   'col' : columns,
                    'Table':'citations_sites'}
             return make_response(jsonify(msg), 400)
 

app/requirements.txt

@@ -2,7 +2,7 @@ requests==2.20.0
 psycopg2-binary
 WSGIserver
 gevent
-pandas
+pandas==0.24.2
 connexion==2.2.0
 Flask==1.0
 Werkzeug==0.15.3

app/server.py

@@ -6,6 +6,7 @@
 import connexion
 import json
 from time import sleep
+from flask import Response
 
 # Create the application instance
 app = connexion.App(__name__, specification_dir="./")

client/client.py

@@ -3,22 +3,21 @@
 #################
 
 import requests
-from flask import Flask, g, render_template,request,redirect,Response,flash,jsonify, make_response
-from flask import request
+from flask import Flask, render_template,request,redirect,Response,flash
 
 import logging
-import json
 from time import sleep
-import sys
-from flask_cors import CORS, cross_origin
+
+from flask_cors import CORS
 
 # Create app
 app = Flask(__name__)
 app.secret_key = 'my unobvious secret key'
 
 # The maximum allowed payload to 16 megabytes.
 app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
-cors = CORS(app, resources={r'/*': {'origins': '*'}})
+cors = CORS(app, resources={r'/*': {'origins': 'http://localhost:3001'}})
+
 logging.getLogger('flask_cors').level = logging.DEBUG
 
 ALLOWED_EXTENSIONS = set(['csv', 'xls','xlsx'])

client/requirements.txt

@@ -1,2 +1,3 @@
 Flask==1.0
 requests==2.20.0
+flask-cors

docker-compose.yml

@@ -48,6 +48,35 @@ services:
       - 6001:6000
     depends_on:
       - yaba_api
+
+  yaba_visualization:
+    build: ./visualization
+    networks:
+      - bety
+    volumes:
+      - './visualization:/code'
+      - '/code/node_modules'
+    ports:
+      - 8008:8075
+    depends_on:
+      - postgres
+
+  yaba_interface:
+    build: ./interface
+    networks: 
+      - bety
+    volumes:
+      - './interface:/code'
+      - '/code/node_modules'
+    ports:
+      - 3001:3000
+    environment:
+      - CHOKIDAR_USEPOLLING=true
+    depends_on:
+      - yaba_client
+      - yaba_visualization
+    stdin_open: true
+
 volumes:
   postgres:
 

interface/.dockerignore

@@ -0,0 +1 @@
+node_modules

interface/Dockerfile

@@ -0,0 +1,16 @@
+FROM node:12.18.2-alpine
+
+COPY . /code
+
+WORKDIR /code
+
+ENV PATH /code/node_modules/.bin:$PATH
+
+RUN npm install --silent
+
+RUN npm install [email protected] -g --silent
+
+EXPOSE 3000
+
+# start app
+CMD ["npm", "start"]

interface/public/index.html

@@ -2,14 +2,14 @@
 <html lang="en">
   <head>
     <meta charset="utf-8" />
-    <link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
+    <link rel="icon" href="%PUBLIC_URL%/bety.png" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <meta name="theme-color" content="#000000" />
     <meta
       name="description"
-      content="Web site created using create-react-app"
+      content="BETYdb-YABA Interface for Metadata Upload"
     />
-    <link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
+    <link rel="apple-touch-icon" href="%PUBLIC_URL%/bety.png" />
     <!--
       manifest.json provides metadata used when your web app is installed on a
       user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
@@ -26,7 +26,7 @@
     -->
     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" />
     <link rel="stylesheet" href="https://unpkg.com/[email protected]/dist/leaflet.css" integrity="sha512-xwE/Az9zrjBIphAcBb3F6JVqxf46+CDLwfLMHloNu6KEQCAWi6HcDUbeOfBIptF7tcCzusKFjFw2yuvEpDL9wQ==" crossorigin="" />
-    <title>React App</title>
+    <title>BETYdb-YABA Interface</title>
   </head>
   <body>
     <noscript>You need to enable JavaScript to run this app.</noscript>

interface/src/components/grid.js

@@ -31,8 +31,8 @@ class Grid extends Component{
                 if(!label.includes(data[i][type])){
                     label.push(data[i][type])
                 }
-            }          
-
+            }
+            
             color= getRandomColors(label.length)
             let plots = new Array();
             let xpos = 1; //starting xpos and ypos at 1 so the stroke will show when we make the grid below

interface/src/components/requests.js

@@ -29,7 +29,7 @@ export function getVisualData(files, type, username){
         params= 'getTreatmentSites'
     }
 
-    let res= fetch(`http://localhost:3001/${params}`,  {
+    let res= fetch(`http://localhost:8008/${params}`,  {
         method: 'POST',
         body: formData,
         redirect: 'follow'
@@ -48,7 +48,7 @@ export function getGeoJSON(file){
 
     formData.append("shp_file", file);
 
-    let res= fetch("http://localhost:3001/getGeoJSON", {
+    let res= fetch("http://localhost:8008/getGeoJSON", {
         method: 'POST',
         body: formData,
         redirect: 'follow'
@@ -98,7 +98,7 @@ function getGeoFile(zip_file, file_type){
     redirect: 'follow'
     };
 
-    const res = fetch("http://localhost:3001/getGeoFile", requestOptions)
+    const res = fetch("http://localhost:8008/getGeoFile", requestOptions)
     .then(response => new File([response.blob()], `${file_type}_file.${file_type}`))
     .catch(error => {
         throw Error(error)

interface/src/setupTests.js

@@ -5,4 +5,4 @@
 import '@testing-library/jest-dom/extend-expect';
 import { configure } from 'enzyme';
 import Adapter from 'enzyme-adapter-react-16';
-configure({ adapter: new Adapter() });
+configure({ adapter: new Adapter() });

visualization/.dockerignore

@@ -0,0 +1 @@
+node_modules

visualization/Dockerfile

@@ -0,0 +1,14 @@
+FROM node:12.18.2-alpine
+
+COPY . /code
+
+WORKDIR /code
+
+ENV PATH /code/node_modules/.bin:$PATH
+
+RUN npm install --silent
+
+EXPOSE 8075
+
+# start app
+CMD ["npm", "start"]