From 8e5dc2ee73401d3de530ae803d91fad44a3e33c4 Mon Sep 17 00:00:00 2001 From: theakhiljha Date: Wed, 31 Mar 2021 12:26:41 +0530 Subject: [PATCH 1/2] checking to generate coverage report --- .github/workflows/py_tests.yml | 3 +- app/Meta.py | 613 +++++++++++++++++++-------------- yaba_test/test_yaba.py | 10 +- 3 files changed, 366 insertions(+), 260 deletions(-) diff --git a/.github/workflows/py_tests.yml b/.github/workflows/py_tests.yml index 1dad83b..71fe853 100644 --- a/.github/workflows/py_tests.yml +++ b/.github/workflows/py_tests.yml @@ -51,4 +51,5 @@ jobs: pytest --cov= ${{ matrix.path }} --cov-report=xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 \ No newline at end of file + uses: codecov/codecov-action@v1 + \ No newline at end of file diff --git a/app/Meta.py b/app/Meta.py index fb5c1ff..74fcec7 100644 --- a/app/Meta.py +++ b/app/Meta.py @@ -7,16 +7,18 @@ import traceback import logging import pandas as pd -from flask import make_response,jsonify +from flask import make_response, jsonify from db import * -from werkzeug import secure_filename,FileStorage +from werkzeug import secure_filename, FileStorage from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import OperationalError from shapely.geometry import Polygon -logging.basicConfig(filename='app.log',format='%(asctime)s %(message)s',level=logging.INFO) +logging.basicConfig( + filename="app.log", format="%(asctime)s %(message)s", level=logging.INFO +) -UPLOAD_FOLDER = 'temp' +UPLOAD_FOLDER = "temp" def save_tempFile(File): @@ -27,12 +29,13 @@ def save_tempFile(File): # Sanitise the filename a_file_name = secure_filename(File.filename) - UPLOAD_PATH=os.path.join(os.getcwd(),UPLOAD_FOLDER,a_file_name) - # Save file + UPLOAD_PATH = os.path.join(os.getcwd(), UPLOAD_FOLDER, a_file_name) + # Save file File.save(UPLOAD_PATH) return None -def insert_experiments(username,fileName,status=True): + +def insert_experiments(username, fileName, status=True): """ This function responds to a request for /yaba/v1/experiments with csv file. It first checks the file is appropriate one and then add new column user_id to dataframe. @@ -46,56 +49,62 @@ def insert_experiments(username,fileName,status=True): 401 Unauthorized | No user exists 410 Default error.See logs for more information """ - + try: - user_id=fetch_id(username, table='users') + user_id = fetch_id(username, table="users") if not user_id: return 401 - #Reading the CSV file into DataFrame - data = pd.read_csv(fileName,delimiter = ',') + # Reading the CSV file into DataFrame + data = pd.read_csv(fileName, delimiter=",") - #Checking necessary columns are there. - columns=data.columns.values.tolist() - accepted_columns=['name','start_date','end_date','description'] - if(all(x in columns for x in accepted_columns)): + # Checking necessary columns are there. + columns = data.columns.values.tolist() + accepted_columns = ["name", "start_date", "end_date", "description"] + if all(x in columns for x in accepted_columns): if status == False: - msg = {'Message' : 'Successfully Validated', - 'Table' : 'Experiments'} - + msg = {"Message": "Successfully Validated", "Table": "Experiments"} + return make_response(jsonify(msg), 200) - - data['user_id']=user_id - insert_table(table='experiments',data=data) - msg = {'Message' : 'Successfully inserted', - 'Table Affected' : 'Experiments', - 'Lines Inserted': data.shape[0]} - + + data["user_id"] = user_id + insert_table(table="experiments", data=data) + msg = { + "Message": "Successfully inserted", + "Table Affected": "Experiments", + "Lines Inserted": data.shape[0], + } + return make_response(jsonify(msg), 201) else: - msg = {'Message' : 'File not acceptable. Check the format of file or columns', - 'Table' : 'Experiments'} + msg = { + "Message": "File not acceptable. Check the format of file or columns", + "Table": "Experiments", + } return make_response(jsonify(msg), 400) - + except OperationalError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Database Conection Error'} + msg = {"Message": "Database Conection Error"} return make_response(jsonify(msg), 500) except IntegrityError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : '(psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint "unique_name_per_species"'} + msg = { + "Message": '(psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint "unique_name_per_species"' + } return make_response(jsonify(msg), 409) except Exception as e: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Error' + str(e)} + msg = {"Message": "Error" + str(e)} return make_response(jsonify(msg), 410) - -def insert_sites(fileName,shp_file,dbf_file,prj_file,shx_file,status=True): + + +def insert_sites(fileName, shp_file, dbf_file, prj_file, shx_file, status=True): """ This function responds to a request for /yaba/v1/sites with csv and shape files.Checks file is appropriate one.Takes all the shape file.Extracts geometry from Geopandas dataframe @@ -114,93 +123,109 @@ def insert_sites(fileName,shp_file,dbf_file,prj_file,shx_file,status=True): 401 Unauthorized | No user exists 410 Default error.See logs for more information """ - try: - #Reading the csv as Dataframe - data = pd.read_csv(fileName,delimiter = ',') - - #Checking necessary columns are there. - columns=data.columns.values.tolist() - accepted_columns=['sitename','city','state','country','notes','greenhouse','geometry','time_zone','soil','soilnotes'] - - #Saving files temporarily.(Will be deleted later) + try: + # Reading the csv as Dataframe + data = pd.read_csv(fileName, delimiter=",") + + # Checking necessary columns are there. + columns = data.columns.values.tolist() + accepted_columns = [ + "sitename", + "city", + "state", + "country", + "notes", + "greenhouse", + "geometry", + "time_zone", + "soil", + "soilnotes", + ] + + # Saving files temporarily.(Will be deleted later) save_tempFile(shp_file) save_tempFile(dbf_file) save_tempFile(prj_file) save_tempFile(shx_file) - #Getting the files path from temp folder - shp_file_target = os.path.join(os.getcwd(),UPLOAD_FOLDER,shp_file.filename) - dbf_file_target = os.path.join(os.getcwd(),UPLOAD_FOLDER,dbf_file.filename) - prj_file_target = os.path.join(os.getcwd(),UPLOAD_FOLDER,prj_file.filename) - shx_file_target = os.path.join(os.getcwd(),UPLOAD_FOLDER,shx_file.filename) + # Getting the files path from temp folder + shp_file_target = os.path.join(os.getcwd(), UPLOAD_FOLDER, shp_file.filename) + dbf_file_target = os.path.join(os.getcwd(), UPLOAD_FOLDER, dbf_file.filename) + prj_file_target = os.path.join(os.getcwd(), UPLOAD_FOLDER, prj_file.filename) + shx_file_target = os.path.join(os.getcwd(), UPLOAD_FOLDER, shx_file.filename) + # Reading the shapefile as DataFrame + data_g1 = gpd.read_file(shp_file_target) - #Reading the shapefile as DataFrame - data_g1=gpd.read_file(shp_file_target) - - data_g = data_g1.to_crs({'init': 'epsg:4326'}) + data_g = data_g1.to_crs({"init": "epsg:4326"}) for index, row in data_g.iterrows(): flat_list = [] - for pt in list(row['geometry'].exterior.coords): + for pt in list(row["geometry"].exterior.coords): if len(pt) < 3: - pt=pt+(115,) + pt = pt + (115,) flat_list.append(pt) poly = Polygon(flat_list) - data.loc[index, 'geometry'] = poly - - if(all(x in columns for x in accepted_columns)): + data.loc[index, "geometry"] = poly + + if all(x in columns for x in accepted_columns): if status == False: - msg = {'Message' : 'Successfully Validated', - 'Table' : 'Sites'} + msg = {"Message": "Successfully Validated", "Table": "Sites"} return make_response(jsonify(msg), 200) - data['time_zone'].fillna("America/Phoenix", inplace = True) - data['soilnotes'].fillna("", inplace = True) - data['greenhouse'].fillna("f", inplace = True) - data=data.fillna('') - file_name='sites_n.csv' - - data.to_csv(file_name, encoding='utf-8', index=False) - - #Inserting in Bety - insert_sites_table(table='sites',data=file_name) - msg = {'Message' : 'Successfully inserted', - 'Table Affected' : 'Sites', - 'Lines Inserted': data_g.shape[0]} + data["time_zone"].fillna("America/Phoenix", inplace=True) + data["soilnotes"].fillna("", inplace=True) + data["greenhouse"].fillna("f", inplace=True) + data = data.fillna("") + file_name = "sites_n.csv" + + data.to_csv(file_name, encoding="utf-8", index=False) + + # Inserting in Bety + insert_sites_table(table="sites", data=file_name) + msg = { + "Message": "Successfully inserted", + "Table Affected": "Sites", + "Lines Inserted": data_g.shape[0], + } os.remove(file_name) - + return make_response(jsonify(msg), 201) else: - msg = {'Message' : 'File not acceptable. Check the format of file or columns', - 'Table' : 'Sites'} - + msg = { + "Message": "File not acceptable. Check the format of file or columns", + "Table": "Sites", + } + return make_response(jsonify(msg), 400) - + except OperationalError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Database Conection Error'} + msg = {"Message": "Database Conection Error"} return make_response(jsonify(msg), 500) except IntegrityError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : '(psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint "unique_name_per_species"'} + msg = { + "Message": '(psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint "unique_name_per_species"' + } return make_response(jsonify(msg), 409) except Exception as e: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Error' + str(e)} + msg = {"Message": "Error" + str(e)} return make_response(jsonify(msg), 410) finally: os.remove(shp_file_target) os.remove(dbf_file_target) os.remove(prj_file_target) - os.remove(shx_file_target) + os.remove(shx_file_target) -def insert_treatments(username,fileName,status=True): + +def insert_treatments(username, fileName, status=True): """ This function responds to a request for /yaba/v1/treatments with csv file.Insertion is done to "treatments" table. @@ -214,55 +239,61 @@ def insert_treatments(username,fileName,status=True): 410 Default error.See logs for more information """ try: - user_id=fetch_id(username, table='users') + user_id = fetch_id(username, table="users") if not user_id: return 401 - #Reading the CSV file into DataFrame - data = pd.read_csv(fileName,delimiter = ',') + # Reading the CSV file into DataFrame + data = pd.read_csv(fileName, delimiter=",") + + # Checking necessary columns are there. + columns = data.columns.values.tolist() + accepted_columns = ["name", "definition", "control"] - #Checking necessary columns are there. - columns=data.columns.values.tolist() - accepted_columns=['name','definition','control'] - - if(all(x in columns for x in accepted_columns)): + if all(x in columns for x in accepted_columns): if status == False: - msg = {'Message' : 'Successfully Validated', - 'Table' : 'Treatments'} + msg = {"Message": "Successfully Validated", "Table": "Treatments"} return make_response(jsonify(msg), 200) - data['user_id']=user_id - insert_table(table='treatments',data=data) - msg = {'Message' : 'Successfully inserted', - 'Table Affected' : 'Treatments', - 'Lines Inserted': data.shape[0]} - + data["user_id"] = user_id + insert_table(table="treatments", data=data) + msg = { + "Message": "Successfully inserted", + "Table Affected": "Treatments", + "Lines Inserted": data.shape[0], + } + return make_response(jsonify(msg), 201) else: - msg = {'Message' : 'File not acceptable. Check the format of file or columns', - 'Table' : 'Treatments'} - + msg = { + "Message": "File not acceptable. Check the format of file or columns", + "Table": "Treatments", + } + return make_response(jsonify(msg), 400) - + except OperationalError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Database Conection Error'} + msg = {"Message": "Database Conection Error"} return make_response(jsonify(msg), 500) except IntegrityError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : '(psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint "unique_name_per_species"'} + msg = { + "Message": '(psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint "unique_name_per_species"' + } return make_response(jsonify(msg), 409) except Exception as e: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Error' + str(e)} + msg = {"Message": "Error" + str(e)} return make_response(jsonify(msg), 410) + def insert_cultivars(fileName, status=True): """ This function responds to a request for /yaba/v1/cultivars @@ -277,62 +308,69 @@ def insert_cultivars(fileName, status=True): 401 Unauthorized | No user exists 410 Default error.See logs for more information """ - try: - data = pd.read_csv(fileName,delimiter = ',') + data = pd.read_csv(fileName, delimiter=",") + + specie_id = fetch_specie_id(data["species"][0]) - specie_id=fetch_specie_id(data['species'][0]) + # Checking necessary columns are there. + columns = data.columns.values.tolist() + accepted_columns = ["name", "species", "ecotype", "notes"] - #Checking necessary columns are there. - columns=data.columns.values.tolist() - accepted_columns=['name','species','ecotype','notes'] - - if(all(x in columns for x in accepted_columns)): + if all(x in columns for x in accepted_columns): if status == False: - msg = {'Message' : 'Successfully Validated', - 'Table' : 'Cultivars'} - + msg = {"Message": "Successfully Validated", "Table": "Cultivars"} + return make_response(jsonify(msg), 200) - new_data = pd.DataFrame(columns=['name', 'specie_id','name','ecotype','notes']) + new_data = pd.DataFrame( + columns=["name", "specie_id", "name", "ecotype", "notes"] + ) + + new_data["name"] = data["name"] + new_data["specie_id"] = specie_id + new_data["ecotype"] = "some text" + new_data["notes"] = "some text" - new_data['name']=data['name'] - new_data['specie_id']=specie_id - new_data['ecotype']='some text' - new_data['notes']='some text' + insert_table(table="cultivars", data=new_data) - insert_table(table='cultivars',data=new_data) + msg = { + "Message": "Successfully inserted", + "Table Affected": "Cultivars", + "Lines Inserted": data.shape[0], + } - msg = {'Message' : 'Successfully inserted', - 'Table Affected' : 'Cultivars', - 'Lines Inserted': data.shape[0]} - return make_response(jsonify(msg), 201) else: - msg = {'Message' : 'File not acceptable. Check the format of file or columns', - 'Table' : 'Cultivars'} - + msg = { + "Message": "File not acceptable. Check the format of file or columns", + "Table": "Cultivars", + } + return make_response(jsonify(msg), 400) - + except OperationalError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Database Conection Error'} + msg = {"Message": "Database Conection Error"} return make_response(jsonify(msg), 500) except IntegrityError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : '(psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint "unique_name_per_species"'} + msg = { + "Message": '(psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint "unique_name_per_species"' + } return make_response(jsonify(msg), 409) except Exception as e: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Error' + str(e)} + msg = {"Message": "Error" + str(e)} return make_response(jsonify(msg), 410) - -def insert_citations(username,fileName, status=True): + + +def insert_citations(username, fileName, status=True): """ This function responds to a request for /yaba/v1/citations with csv file @@ -346,53 +384,69 @@ def insert_citations(username,fileName, status=True): 410 Default error.See logs for more information """ try: - user_id=fetch_id(username, table='users') + user_id = fetch_id(username, table="users") if not user_id: return 401 - data = pd.read_csv(fileName,delimiter = ',') - #Checking necessary columns are there. - columns=data.columns.values.tolist() - accepted_columns=['author','year','title','journal','vol','pg','url','pdf','doi'] - - if(all(x in columns for x in accepted_columns)): + data = pd.read_csv(fileName, delimiter=",") + # Checking necessary columns are there. + columns = data.columns.values.tolist() + accepted_columns = [ + "author", + "year", + "title", + "journal", + "vol", + "pg", + "url", + "pdf", + "doi", + ] + + if all(x in columns for x in accepted_columns): if status == False: - msg = {'Message' : 'Successfully Validated', - 'Table' : 'Citations'} - + msg = {"Message": "Successfully Validated", "Table": "Citations"} + return make_response(jsonify(msg), 200) - #Reading the CSV file into DataFrame - data['user_id']=user_id - insert_table(table='citations',data=data) - msg = {'Message' : 'Successfully inserted', - 'Table Affected' : 'Citations', - 'Lines Inserted': data.shape[0]} - + # Reading the CSV file into DataFrame + data["user_id"] = user_id + insert_table(table="citations", data=data) + msg = { + "Message": "Successfully inserted", + "Table Affected": "Citations", + "Lines Inserted": data.shape[0], + } + return make_response(jsonify(msg), 201) else: - msg = {'Message' : 'File not acceptable. Check the format of file or columns', - 'Table' : 'Citations'} + msg = { + "Message": "File not acceptable. Check the format of file or columns", + "Table": "Citations", + } return make_response(jsonify(msg), 400) - + except OperationalError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Database Conection Error'} + msg = {"Message": "Database Conection Error"} return make_response(jsonify(msg), 500) except IntegrityError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : '(psycopg2.errors.UniqueViolation) duplicate key value violation.'} + msg = { + "Message": "(psycopg2.errors.UniqueViolation) duplicate key value violation." + } return make_response(jsonify(msg), 409) except Exception as e: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Error' + str(e)} + msg = {"Message": "Error" + str(e)} return make_response(jsonify(msg), 410) - + + def insert_experimentSites(fileName, status=True): """ This function responds to a request for /yaba/v1/experiments_sites @@ -408,50 +462,64 @@ def insert_experimentSites(fileName, status=True): 410 Default error.See logs for more information """ try: - data = pd.read_csv(fileName,delimiter = ',') - #Checking necessary columns are there. - columns=data.columns.values.tolist() - accepted_columns=['experiment_name','sitename'] - - if(all(x in columns for x in accepted_columns)): + data = pd.read_csv(fileName, delimiter=",") + # Checking necessary columns are there. + columns = data.columns.values.tolist() + accepted_columns = ["experiment_name", "sitename"] + + if all(x in columns for x in accepted_columns): if status == False: - msg = {'Message' : 'Successfully Validated', - 'Table' : 'Experiments_sites'} - + msg = { + "Message": "Successfully Validated", + "Table": "Experiments_sites", + } + return make_response(jsonify(msg), 200) - - new_data = pd.DataFrame(columns=['experiment_id', 'site_id']) - new_data['experiment_id'] = data.apply(lambda row: fetch_id(row['experiment_name'],table='experiments'), axis=1) - new_data['site_id'] = data.apply(lambda row: fetch_sites_id(row['sitename']), axis=1) - insert_table(table='experiments_sites',data=new_data) - msg = {'Message' : 'Successfully inserted', - 'Table Affected' : 'Experiments_sites', - 'Lines Inserted': data.shape[0]} - + + new_data = pd.DataFrame(columns=["experiment_id", "site_id"]) + new_data["experiment_id"] = data.apply( + lambda row: fetch_id(row["experiment_name"], table="experiments"), + axis=1, + ) + new_data["site_id"] = data.apply( + lambda row: fetch_sites_id(row["sitename"]), axis=1 + ) + insert_table(table="experiments_sites", data=new_data) + msg = { + "Message": "Successfully inserted", + "Table Affected": "Experiments_sites", + "Lines Inserted": data.shape[0], + } + return make_response(jsonify(msg), 201) else: - msg = {'Message' : 'File not acceptable. Check the format of file or columns', - 'Table' : 'Experiments_sites'} + msg = { + "Message": "File not acceptable. Check the format of file or columns", + "Table": "Experiments_sites", + } return make_response(jsonify(msg), 400) - + except OperationalError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Database Conection Error'} + msg = {"Message": "Database Conection Error"} return make_response(jsonify(msg), 500) except IntegrityError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : '(psycopg2.errors.UniqueViolation) duplicate key value violation.'} + msg = { + "Message": "(psycopg2.errors.UniqueViolation) duplicate key value violation." + } return make_response(jsonify(msg), 409) except Exception as e: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Error' + str(e)} + msg = {"Message": "Error" + str(e)} return make_response(jsonify(msg), 410) + def insert_experimentTreatments(fileName, status=True): """ This function responds to a request for /yaba/v1/experiments_treatments @@ -467,52 +535,66 @@ def insert_experimentTreatments(fileName, status=True): 410 Default error.See logs for more information """ try: - - data = pd.read_csv(fileName,delimiter = ',') - #Checking necessary columns are there. - columns=data.columns.values.tolist() - accepted_columns=['experiment_name','treatment_name'] + data = pd.read_csv(fileName, delimiter=",") - if(all(x in columns for x in accepted_columns)): + # Checking necessary columns are there. + columns = data.columns.values.tolist() + accepted_columns = ["experiment_name", "treatment_name"] + + if all(x in columns for x in accepted_columns): if status == False: - msg = {'Message' : 'Successfully Validated', - 'Table' : 'Experiments_treatments'} - + msg = { + "Message": "Successfully Validated", + "Table": "Experiments_treatments", + } + return make_response(jsonify(msg), 200) - - new_data = pd.DataFrame(columns=['experiment_id', 'treatment_id']) - new_data['experiment_id'] = data.apply(lambda row: fetch_id(row['experiment_name'],table='experiments'), axis=1) - new_data['treatment_id'] = data.apply(lambda row: fetch_id(row['treatment_name'],table='treatments'), axis=1) - insert_table(table='experiments_treatments',data=new_data) - msg = {'Message' : 'Successfully inserted', - 'Table Affected' : 'Experiments_treatments', - 'Lines Inserted': data.shape[0]} - + + new_data = pd.DataFrame(columns=["experiment_id", "treatment_id"]) + new_data["experiment_id"] = data.apply( + lambda row: fetch_id(row["experiment_name"], table="experiments"), + axis=1, + ) + new_data["treatment_id"] = data.apply( + lambda row: fetch_id(row["treatment_name"], table="treatments"), axis=1 + ) + insert_table(table="experiments_treatments", data=new_data) + msg = { + "Message": "Successfully inserted", + "Table Affected": "Experiments_treatments", + "Lines Inserted": data.shape[0], + } + return make_response(jsonify(msg), 201) else: - msg = {'Message' : 'File not acceptable. Check the format of file or columns', - 'Table' : 'Experiments_treatments'} + msg = { + "Message": "File not acceptable. Check the format of file or columns", + "Table": "Experiments_treatments", + } return make_response(jsonify(msg), 400) - + except OperationalError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Database Conection Error'} + msg = {"Message": "Database Conection Error"} return make_response(jsonify(msg), 500) except IntegrityError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : '(psycopg2.errors.UniqueViolation) duplicate key value violation.'} + msg = { + "Message": "(psycopg2.errors.UniqueViolation) duplicate key value violation." + } return make_response(jsonify(msg), 409) except Exception as e: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Error' + str(e)} + msg = {"Message": "Error" + str(e)} return make_response(jsonify(msg), 410) + def insert_sitesCultivars(fileName, status=True): """ This function responds to a request for /yaba/v1/sites_cultivars @@ -528,51 +610,62 @@ def insert_sitesCultivars(fileName, status=True): 410 Default error.See logs for more information """ try: - data = pd.read_csv(fileName,delimiter = ',') + data = pd.read_csv(fileName, delimiter=",") - #Checking necessary columns are there. - columns=data.columns.values.tolist() - accepted_columns=['sitename','cultivar_name','specie_id'] + # Checking necessary columns are there. + columns = data.columns.values.tolist() + accepted_columns = ["sitename", "cultivar_name", "specie_id"] - if(all(x in columns for x in accepted_columns)): + if all(x in columns for x in accepted_columns): if status == False: - msg = {'Message' : 'Successfully Validated', - 'Table' : 'Sites_cultivars'} - + msg = {"Message": "Successfully Validated", "Table": "Sites_cultivars"} + return make_response(jsonify(msg), 200) - new_data = pd.DataFrame(columns=['site_id', 'cultivar_id']) - new_data['site_id'] = data.apply(lambda row: fetch_sites_id(row['sitename']), axis=1) - new_data['cultivar_id'] = data.apply(lambda row: fetch_cultivars_id(row['cultivar_name'],row['specie_id']), axis=1) - insert_table(table='sites_cultivars',data=new_data) - msg = {'Message' : 'Successfully inserted', - 'Table Affected' : 'Sites_cultivars', - 'Lines Inserted': data.shape[0]} - + new_data = pd.DataFrame(columns=["site_id", "cultivar_id"]) + new_data["site_id"] = data.apply( + lambda row: fetch_sites_id(row["sitename"]), axis=1 + ) + new_data["cultivar_id"] = data.apply( + lambda row: fetch_cultivars_id(row["cultivar_name"], row["specie_id"]), + axis=1, + ) + insert_table(table="sites_cultivars", data=new_data) + msg = { + "Message": "Successfully inserted", + "Table Affected": "Sites_cultivars", + "Lines Inserted": data.shape[0], + } + return make_response(jsonify(msg), 201) else: - msg = {'Message' : 'File not acceptable. Check the format of file or columns', - 'Table' : 'Sites_cultivars'} + msg = { + "Message": "File not acceptable. Check the format of file or columns", + "Table": "Sites_cultivars", + } return make_response(jsonify(msg), 400) - + except OperationalError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Database Conection Error or Inconsistent Data'} + msg = {"Message": "Database Conection Error or Inconsistent Data"} return make_response(jsonify(msg), 500) except IntegrityError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : '(psycopg2.errors.UniqueViolation) duplicate key value violation.'} + msg = { + "Message": "(psycopg2.errors.UniqueViolation) duplicate key value violation." + } return make_response(jsonify(msg), 409) except Exception as e: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Error' + str(e)} + msg = {"Message": "Error" + str(e)} return make_response(jsonify(msg), 410) - + + def insert_citationsSites(fileName, status=True): """ This function responds to a request for /yaba/v1/citations_sites @@ -587,49 +680,61 @@ def insert_citationsSites(fileName, status=True): 401 Unauthorized | No user exists 410 Default error.See logs for more information """ - + try: - data = pd.read_csv(fileName,delimiter = ',') + data = pd.read_csv(fileName, delimiter=",") - #Checking necessary columns are there. - columns=data.columns.values.tolist() - accepted_columns=['author','year','title','sitename'] + # Checking necessary columns are there. + columns = data.columns.values.tolist() + accepted_columns = ["author", "year", "title", "sitename"] - if(all(x in columns for x in accepted_columns)): + if all(x in columns for x in accepted_columns): if status == False: - msg = {'Message' : 'Successfully Validated', - 'Table' : 'Citations_sites'} - + msg = {"Message": "Successfully Validated", "Table": "Citations_sites"} + return make_response(jsonify(msg), 200) - - new_data = pd.DataFrame(columns=['citation_id','site_id']) - new_data['site_id'] = data.apply(lambda row: fetch_sites_id(row['sitename']), axis=1) - new_data['citation_id'] = data.apply(lambda row: fetch_citations_id(row['author'],row['year'],row['title']), axis=1) - - insert_table(table='citations_sites',data=new_data) - msg = {'Message' : 'Successfully inserted', - 'Table Affected' : 'Citations_sites', - 'Lines Inserted': data.shape[0]} - + + new_data = pd.DataFrame(columns=["citation_id", "site_id"]) + new_data["site_id"] = data.apply( + lambda row: fetch_sites_id(row["sitename"]), axis=1 + ) + new_data["citation_id"] = data.apply( + lambda row: fetch_citations_id( + row["author"], row["year"], row["title"] + ), + axis=1, + ) + + insert_table(table="citations_sites", data=new_data) + msg = { + "Message": "Successfully inserted", + "Table Affected": "Citations_sites", + "Lines Inserted": data.shape[0], + } + return make_response(jsonify(msg), 201) else: - msg = {'Message' : 'File not acceptable and Check the format of file or columns', - 'Table':'Citations_sites'} + msg = { + "Message": "File not acceptable and Check the format of file or columns", + "Table": "Citations_sites", + } return make_response(jsonify(msg), 400) - + except OperationalError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Database Conection Error'} + msg = {"Message": "Database Conection Error"} return make_response(jsonify(msg), 500) except IntegrityError: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : '(psycopg2.errors.UniqueViolation) duplicate key value violation.'} + msg = { + "Message": "(psycopg2.errors.UniqueViolation) duplicate key value violation." + } return make_response(jsonify(msg), 409) except Exception as e: # Logs the error appropriately logging.error(traceback.format_exc()) - msg = {'Message' : 'Error' + str(e)} + msg = {"Message": "Error" + str(e)} return make_response(jsonify(msg), 410) \ No newline at end of file diff --git a/yaba_test/test_yaba.py b/yaba_test/test_yaba.py index e8a1773..9eb2d19 100644 --- a/yaba_test/test_yaba.py +++ b/yaba_test/test_yaba.py @@ -52,7 +52,7 @@ def test_insert_cultivars(self): csv = 'input_files/'+csv_filename files = {'fileName': open(csv, 'rb')} response = requests.post( - 'http://0.0.0.0:5001/yaba/v1/cultivars', files=files) + 'http://0.0.0.0:5001/yaba/v1/cultivars?username=guestuser', files=files) self.assertEqual(response.status_code, 201) self.assertTrue(response.ok) @@ -70,7 +70,7 @@ def test_insert_experimentSites(self): csv = 'input_files/'+csv_filename files = {'fileName': open(csv, 'rb')} response = requests.post( - 'http://0.0.0.0:5001/yaba/v1/experiments_sites', files=files) + 'http://0.0.0.0:5001/yaba/v1/experiments_sites?username=guestuser', files=files) self.assertEqual(response.status_code, 201) self.assertTrue(response.ok) @@ -79,7 +79,7 @@ def test_insert_experimentTreatments(self): csv = 'input_files/'+csv_filename files = {'fileName': open(csv, 'rb')} response = requests.post( - 'http://0.0.0.0:5001/yaba/v1/experiments_treatments', files=files) + 'http://0.0.0.0:5001/yaba/v1/experiments_treatments?username=guestuser', files=files) self.assertEqual(response.status_code, 201) self.assertTrue(response.ok) @@ -88,7 +88,7 @@ def test_insert_sitesCultivars(self): csv = 'input_files/'+csv_filename files = {'fileName': open(csv, 'rb')} response = requests.post( - 'http://0.0.0.0:5001/yaba/v1/sites_cultivars', files=files) + 'http://0.0.0.0:5001/yaba/v1/sites_cultivars?username=guestuser', files=files) self.assertEqual(response.status_code, 201) self.assertTrue(response.ok) @@ -97,7 +97,7 @@ def test_insert_citationsSites(self): csv = 'input_files/'+csv_filename files = {'fileName': open(csv, 'rb')} response = requests.post( - 'http://0.0.0.0:5001/yaba/v1/citations_sites', files=files) + 'http://0.0.0.0:5001/yaba/v1/citations_sites?username=guestuser', files=files) self.assertEqual(response.status_code, 201) self.assertTrue(response.ok) From b14e15c03d5d908ab27805666eb910dbe2ffdbca Mon Sep 17 00:00:00 2001 From: theakhiljha Date: Wed, 31 Mar 2021 12:39:25 +0530 Subject: [PATCH 2/2] Fixed Pylint issues --- app/db.py | 137 ++++++++------- app/server.py | 14 +- app/setup/setup_environment.py | 37 ++-- client/client.py | 245 +++++++++++++++------------ yaba_client_test/test_yaba_client.py | 108 ++++++------ yaba_test/test_yaba.py | 114 +++++++------ 6 files changed, 352 insertions(+), 303 deletions(-) diff --git a/app/db.py b/app/db.py index f09455c..3d363d2 100644 --- a/app/db.py +++ b/app/db.py @@ -3,84 +3,93 @@ from sqlalchemy.sql import text from setup import setup_environment -def insert_table(table,data): - #Make PostgreSQL Connection - engine = setup_environment.get_database() - data.to_sql(table, engine, if_exists = 'append', index=False) -def insert_sites_table(table,data): - #Make PostgreSQL Connection - csv.field_size_limit(sys.maxsize) - engine = setup_environment.get_database() - connection = None - connection = engine.raw_connection() - cur = connection.cursor() - with open(data, 'r') as f: - reader = csv.reader(f) - next(reader) # Skip the header row. - for row in reader: - cur.execute("INSERT INTO sites (sitename,city,state,country,notes,greenhouse,geometry,time_zone,soil,soilnotes) VALUES (%s, %s, %s, %s,%s, %s, ST_MakeValid(ST_Geomfromtext(%s,4326)), %s,%s, %s)",row) - print(row) - connection.commit() - connection.close() - +def insert_table(table, data): + # Make PostgreSQL Connection + engine = setup_environment.get_database() + data.to_sql(table, engine, if_exists="append", index=False) + + +def insert_sites_table(table, data): + # Make PostgreSQL Connection + csv.field_size_limit(sys.maxsize) + engine = setup_environment.get_database() + connection = None + connection = engine.raw_connection() + cur = connection.cursor() + with open(data, "r") as f: + reader = csv.reader(f) + next(reader) # Skip the header row. + for row in reader: + cur.execute( + "INSERT INTO sites (sitename,city,state,country,notes,greenhouse,geometry,time_zone,soil,soilnotes) VALUES (%s, %s, %s, %s,%s, %s, ST_MakeValid(ST_Geomfromtext(%s,4326)), %s,%s, %s)", + row, + ) + print(row) + connection.commit() + connection.close() + def fetch_specie_id(species): # Make PostgreSQL Connection - engine = setup_environment.get_database() - connection = None - connection = engine.connect() - query = 'select id from species where scientificname = :name' - result_set = connection.execute(text(query), name = species).fetchone() - connection.close() - for r in result_set: - return r + engine = setup_environment.get_database() + connection = None + connection = engine.connect() + query = "select id from species where scientificname = :name" + result_set = connection.execute(text(query), name=species).fetchone() + connection.close() + for r in result_set: + return r def fetch_sites_id(sitename): # Make PostgreSQL Connection - engine = setup_environment.get_database() - connection = None - connection = engine.connect() - query = 'select id from sites where sitename = :name' - result_set = connection.execute(text(query), name = sitename).fetchone() - connection.close() - for r in result_set: - return r + engine = setup_environment.get_database() + connection = None + connection = engine.connect() + query = "select id from sites where sitename = :name" + result_set = connection.execute(text(query), name=sitename).fetchone() + connection.close() + for r in result_set: + return r -def fetch_cultivars_id(name,specie_id): +def fetch_cultivars_id(name, specie_id): # Make PostgreSQL Connection - engine = setup_environment.get_database() - connection = None - connection = engine.connect() - query = 'select id from cultivars where name = :name and specie_id = :specie_id' - result_set = connection.execute(text(query), name = name,specie_id=specie_id).fetchone() - connection.close() - for r in result_set: - return r + engine = setup_environment.get_database() + connection = None + connection = engine.connect() + query = "select id from cultivars where name = :name and specie_id = :specie_id" + result_set = connection.execute( + text(query), name=name, specie_id=specie_id + ).fetchone() + connection.close() + for r in result_set: + return r -def fetch_citations_id(author,year,title): +def fetch_citations_id(author, year, title): # Make PostgreSQL Connection - engine = setup_environment.get_database() - connection = None - connection = engine.connect() - query = 'select id from citations where author = :author and year = :year and title = :title' - result_set = connection.execute(text(query),author=author,year=year, title = title).fetchone() - connection.close() - for r in result_set: - return r + engine = setup_environment.get_database() + connection = None + connection = engine.connect() + query = "select id from citations where author = :author and year = :year and title = :title" + result_set = connection.execute( + text(query), author=author, year=year, title=title + ).fetchone() + connection.close() + for r in result_set: + return r -#Fetching ID from Users,experiments,cultivars and treatments tables -def fetch_id(value,table): +# Fetching ID from Users,experiments,cultivars and treatments tables +def fetch_id(value, table): # Make PostgreSQL Connection - connection = None - engine = setup_environment.get_database() - connection = engine.connect() - query = 'select id from '+table+' where name = :name' - result_set = connection.execute(text(query), name = value).fetchone() - connection.close() - for r in result_set: - return r \ No newline at end of file + connection = None + engine = setup_environment.get_database() + connection = engine.connect() + query = "select id from " + table + " where name = :name" + result_set = connection.execute(text(query), name=value).fetchone() + connection.close() + for r in result_set: + return r diff --git a/app/server.py b/app/server.py index 063cd9e..08ef93c 100644 --- a/app/server.py +++ b/app/server.py @@ -2,7 +2,7 @@ Main module of the Server file """ -#Importing moudles +# Importing moudles import connexion import json from time import sleep @@ -14,7 +14,7 @@ sleep(5) # Read the swagger.yml file to configure the endpoints -app.add_api("yaba.yaml",validate_responses=False) +app.add_api("yaba.yaml", validate_responses=False) # create a URL route in our application for "/" @@ -26,10 +26,12 @@ def home(): (on docker: localhost:5001/) :return: the below json message """ - return Response(json.dumps("Welcome to YABA API Index Route"), mimetype='application/json') + return Response( + json.dumps("Welcome to YABA API Index Route"), mimetype="application/json" + ) if __name__ == "__main__": - app.run(host="0.0.0.0",port=5000,debug=False) - #http_server = WSGIServer(('0.0.0.0', 5000), app) - #http_server.serve_forever() + app.run(host="0.0.0.0", port=5000, debug=False) + # http_server = WSGIServer(('0.0.0.0', 5000), app) + # http_server.serve_forever() diff --git a/app/setup/setup_environment.py b/app/setup/setup_environment.py index c18aebc..fc5331c 100644 --- a/app/setup/setup_environment.py +++ b/app/setup/setup_environment.py @@ -14,12 +14,12 @@ def get_database(): log.info("Successfully Connected to BETY Database!") except IOError: log.exception("Failed to get database connection!") - return None, 'fail' + return None, "fail" return engine -def get_connection_from_config(config_file_name='credentials.yaml'): +def get_connection_from_config(config_file_name="credentials.yaml"): """ Sets up database connection from config file. Input: @@ -28,19 +28,25 @@ def get_connection_from_config(config_file_name='credentials.yaml'): credentials for the PostgreSQL database """ - with open(config_file_name, 'r') as f: + with open(config_file_name, "r") as f: creds = yaml.safe_load(f) - if not ('PGHOST' in creds.keys() and - 'PGUSER' in creds.keys() and - 'PGPASSWORD' in creds.keys() and - 'PGDATABASE' in creds.keys() and - 'PGPORT' in creds.keys()): - raise Exception('Bad config file: ' + config_file_name) + if not ( + "PGHOST" in creds.keys() + and "PGUSER" in creds.keys() + and "PGPASSWORD" in creds.keys() + and "PGDATABASE" in creds.keys() + and "PGPORT" in creds.keys() + ): + raise Exception("Bad config file: " + config_file_name) - return get_engine(creds['PGDATABASE'], creds['PGUSER'], - creds['PGHOST'], creds['PGPORT'], - creds['PGPASSWORD']) + return get_engine( + creds["PGDATABASE"], + creds["PGUSER"], + creds["PGHOST"], + creds["PGPORT"], + creds["PGPASSWORD"], + ) def get_engine(db, user, host, port, passwd): @@ -54,7 +60,8 @@ def get_engine(db, user, host, port, passwd): passwd: Password for the database """ - url = 'postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{db}'.format( - user=user, passwd=passwd, host=host, port=port, db=db) - engine = create_engine(url,poolclass=QueuePool) + url = "postgresql+psycopg2://{user}:{passwd}@{host}:{port}/{db}".format( + user=user, passwd=passwd, host=host, port=port, db=db + ) + engine = create_engine(url, poolclass=QueuePool) return engine diff --git a/client/client.py b/client/client.py index 61d4391..ae6a2a7 100644 --- a/client/client.py +++ b/client/client.py @@ -3,7 +3,7 @@ ################# import requests -from flask import Flask, render_template,request,redirect,Response,flash +from flask import Flask, render_template, request, redirect, Response, flash import logging from time import sleep @@ -12,163 +12,182 @@ # Create app app = Flask(__name__) -app.secret_key = 'my unobvious secret key' +app.secret_key = "my unobvious secret key" # The maximum allowed payload to 16 megabytes. -app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024 -cors = CORS(app, resources={r'/*': {'origins': 'http://localhost:3001'}}) +app.config["MAX_CONTENT_LENGTH"] = 16 * 1024 * 1024 +cors = CORS(app, resources={r"/*": {"origins": "http://localhost:3001"}}) -logging.getLogger('flask_cors').level = logging.DEBUG -logging.basicConfig(filename='client.log',format='%(asctime)s %(message)s',level=logging.INFO) +logging.getLogger("flask_cors").level = logging.DEBUG +logging.basicConfig( + filename="client.log", format="%(asctime)s %(message)s", level=logging.INFO +) + +ALLOWED_EXTENSIONS = set(["csv", "xls", "xlsx"]) -ALLOWED_EXTENSIONS = set(['csv', 'xls','xlsx']) @app.route("/") -def index(): +def index(): return render_template("index.html") -@app.route('/experiments',methods = ['POST']) -def experiments(): - if request.method == 'POST': - response=postRequest(request,'experiments') - return Response(response.text,mimetype="application/json") - -@app.route('/sites',methods = ['POST']) -def sites(): - if request.method == 'POST': - response=postRequest_with_shpFile(request,'sites') - return Response(response.text,mimetype="application/json") - - -@app.route('/treatments',methods = ['POST']) -def treatments(): - if request.method == 'POST': - response=postRequest(request,'treatments') - return Response(response.text,mimetype="application/json") - -@app.route('/cultivars',methods = ['POST']) -def cultivars(): - if request.method == 'POST': - response=postRequest(request,'cultivars') - return Response(response.text,mimetype="application/json") - -@app.route('/citations',methods = ['POST']) -def citations(): - if request.method == 'POST': - response=postRequest(request,'citations') - return Response(response.text,mimetype="application/json") - -@app.route('/experiments_sites',methods = ['POST']) -def experiments_sites(): - if request.method == 'POST': - response=postRequest(request,'experiments_sites') - return Response(response.text,mimetype="application/json") - -@app.route('/experiments_treatments',methods = ['POST']) -def experiments_treatments(): - if request.method == 'POST': - response=postRequest(request,'experiments_treatments') - return Response(response.text,mimetype="application/json") - -@app.route('/sites_cultivars',methods = ['POST']) -def sites_cultivars(): - if request.method == 'POST': - response=postRequest(request,'sites_cultivars') - return Response(response.text,mimetype="application/json") - -@app.route('/citations_sites',methods = ['POST']) -def citations_sites(): - if request.method == 'POST': - response=postRequest(request,'citations_sites') - return Response(response.text,mimetype="application/json") + +@app.route("/experiments", methods=["POST"]) +def experiments(): + if request.method == "POST": + response = postRequest(request, "experiments") + return Response(response.text, mimetype="application/json") + + +@app.route("/sites", methods=["POST"]) +def sites(): + if request.method == "POST": + response = postRequest_with_shpFile(request, "sites") + return Response(response.text, mimetype="application/json") + + +@app.route("/treatments", methods=["POST"]) +def treatments(): + if request.method == "POST": + response = postRequest(request, "treatments") + return Response(response.text, mimetype="application/json") + + +@app.route("/cultivars", methods=["POST"]) +def cultivars(): + if request.method == "POST": + response = postRequest(request, "cultivars") + return Response(response.text, mimetype="application/json") + + +@app.route("/citations", methods=["POST"]) +def citations(): + if request.method == "POST": + response = postRequest(request, "citations") + return Response(response.text, mimetype="application/json") + + +@app.route("/experiments_sites", methods=["POST"]) +def experiments_sites(): + if request.method == "POST": + response = postRequest(request, "experiments_sites") + return Response(response.text, mimetype="application/json") + + +@app.route("/experiments_treatments", methods=["POST"]) +def experiments_treatments(): + if request.method == "POST": + response = postRequest(request, "experiments_treatments") + return Response(response.text, mimetype="application/json") + + +@app.route("/sites_cultivars", methods=["POST"]) +def sites_cultivars(): + if request.method == "POST": + response = postRequest(request, "sites_cultivars") + return Response(response.text, mimetype="application/json") + + +@app.route("/citations_sites", methods=["POST"]) +def citations_sites(): + if request.method == "POST": + response = postRequest(request, "citations_sites") + return Response(response.text, mimetype="application/json") + @app.errorhandler(404) def page_not_found(e): return render_template("404.html"), 404 + @app.errorhandler(500) def server_error(e): return render_template("500.html"), 500 + def allowed_file(filename): - return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS + return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS + def _url(path): - return 'http://yaba_api:5000/yaba/v1/' + path + return "http://yaba_api:5000/yaba/v1/" + path -def postRequest(request,endpoint): - if 'fileName' not in request.files: - flash('No file part') + +def postRequest(request, endpoint): + if "fileName" not in request.files: + flash("No file part") return redirect(request.url) - file = request.files['fileName'] + file = request.files["fileName"] if not file.filename: - flash('No file selected for uploading') + flash("No file selected for uploading") return redirect(request.url) if file and allowed_file(file.filename): - payload = {'fileName':file} - flash('File successfully uploaded') + payload = {"fileName": file} + flash("File successfully uploaded") - if 'username' not in request.args: - if 'status' not in request.args: + if "username" not in request.args: + if "status" not in request.args: sleep(1) - response = requests.post(_url(endpoint),files=payload) - + response = requests.post(_url(endpoint), files=payload) + else: - status=request.args['status'] - params={'status':status} + status = request.args["status"] + params = {"status": status} sleep(1) - response = requests.post(_url(endpoint),files=payload,params=params) - + response = requests.post(_url(endpoint), files=payload, params=params) + else: - username=request.args['username'] - if 'status' not in request.args: - params={'username':username} + username = request.args["username"] + if "status" not in request.args: + params = {"username": username} sleep(1) - response = requests.post(_url(endpoint),files=payload,params=params) - + response = requests.post(_url(endpoint), files=payload, params=params) + else: - status=request.args['status'] - params={'username':username, 'status':status} + status = request.args["status"] + params = {"username": username, "status": status} sleep(1) - response = requests.post(_url(endpoint),files=payload,params=params) - + response = requests.post(_url(endpoint), files=payload, params=params) + return response - + else: - flash('Allowed file types are csv, xls, xlsx') + flash("Allowed file types are csv, xls, xlsx") return redirect(request.url) -def postRequest_with_shpFile(request,endpoint): - if 'fileName' not in request.files: - flash('No file part') + +def postRequest_with_shpFile(request, endpoint): + if "fileName" not in request.files: + flash("No file part") return redirect(request.url) - file = request.files['fileName'] - shp_file = request.files['shp_file'] - dbf_file = request.files['dbf_file'] - prj_file = request.files['prj_file'] - shx_file = request.files['shx_file'] - + file = request.files["fileName"] + shp_file = request.files["shp_file"] + dbf_file = request.files["dbf_file"] + prj_file = request.files["prj_file"] + shx_file = request.files["shx_file"] + if not file.filename: - flash('No file selected for uploading') + flash("No file selected for uploading") return redirect(request.url) if file and allowed_file(file.filename): - #payload = {'fileName':file} - payload= {'fileName': file, - 'shp_file' : shp_file, - 'dbf_file' : dbf_file, - 'prj_file' : prj_file, - 'shx_file' : shx_file } - - flash('File successfully uploaded') + # payload = {'fileName':file} + payload = { + "fileName": file, + "shp_file": shp_file, + "dbf_file": dbf_file, + "prj_file": prj_file, + "shx_file": shx_file, + } + + flash("File successfully uploaded") sleep(1) - response = requests.post(_url(endpoint),files=payload) + response = requests.post(_url(endpoint), files=payload) return response - + else: - flash('Allowed file types are csv, xls, xlsx') + flash("Allowed file types are csv, xls, xlsx") return redirect(request.url) -if __name__ == '__main__': - app.run(host="0.0.0.0",port=6000,debug=True) +if __name__ == "__main__": + app.run(host="0.0.0.0", port=6000, debug=True) diff --git a/yaba_client_test/test_yaba_client.py b/yaba_client_test/test_yaba_client.py index 1a0f145..c79f9a4 100644 --- a/yaba_client_test/test_yaba_client.py +++ b/yaba_client_test/test_yaba_client.py @@ -4,100 +4,100 @@ import time -BASE_URL = 'http://0.0.0.0:6001/' +BASE_URL = "http://0.0.0.0:6001/" class BasicTests(unittest.TestCase): - def test_insert_experiments(self): - csv_filename = 'experiments.csv' - csv_path = 'input_files/experiments.csv' - files = {'fileName': open(csv_path, 'rb')} + csv_filename = "experiments.csv" + csv_path = "input_files/experiments.csv" + files = {"fileName": open(csv_path, "rb")} response = requests.post( - 'http://0.0.0.0:6001/experiments?username=guestuser', files=files) + "http://0.0.0.0:6001/experiments?username=guestuser", files=files + ) self.assertEqual(response.status_code, 200) self.assertTrue(response.ok) def test_insert_sites(self): - csv_filename = 'sites.csv' - csv_path = 'input_files/'+csv_filename - - shp_file = 'input_files/S8_two_row_polys.shp' - dbf_file = 'input_files/S8_two_row_polys.dbf' - prj_file = 'input_files/S8_two_row_polys.prj' - shx_file = 'input_files/S8_two_row_polys.shx' - - files= {'fileName': open(csv_path, 'rb'), - 'shp_file': open(shp_file, 'rb'), - 'dbf_file': open(dbf_file, 'rb'), - 'prj_file': open(prj_file, 'rb'), - 'shx_file': open(shx_file, 'rb')} - - response = requests.post('http://0.0.0.0:6001/sites', - files=files) + csv_filename = "sites.csv" + csv_path = "input_files/" + csv_filename + + shp_file = "input_files/S8_two_row_polys.shp" + dbf_file = "input_files/S8_two_row_polys.dbf" + prj_file = "input_files/S8_two_row_polys.prj" + shx_file = "input_files/S8_two_row_polys.shx" + + files = { + "fileName": open(csv_path, "rb"), + "shp_file": open(shp_file, "rb"), + "dbf_file": open(dbf_file, "rb"), + "prj_file": open(prj_file, "rb"), + "shx_file": open(shx_file, "rb"), + } + + response = requests.post("http://0.0.0.0:6001/sites", files=files) self.assertEqual(response.status_code, 200) self.assertTrue(response.ok) def test_insert_citations(self): - csv_filename = 'citations.csv' - csv_path = 'input_files/citations.csv' - files = {'fileName': open(csv_path, 'rb')} + csv_filename = "citations.csv" + csv_path = "input_files/citations.csv" + files = {"fileName": open(csv_path, "rb")} response = requests.post( - 'http://0.0.0.0:6001/citations?username=guestuser', files=files) + "http://0.0.0.0:6001/citations?username=guestuser", files=files + ) self.assertEqual(response.status_code, 200) self.assertTrue(response.ok) def test_insert_treatments(self): - csv_filename = 'treatments.csv' - csv = 'input_files/'+csv_filename - files = {'fileName': open(csv, 'rb')} + csv_filename = "treatments.csv" + csv = "input_files/" + csv_filename + files = {"fileName": open(csv, "rb")} response = requests.post( - 'http://0.0.0.0:6001/treatments?username=guestuser', files=files) + "http://0.0.0.0:6001/treatments?username=guestuser", files=files + ) self.assertEqual(response.status_code, 200) self.assertTrue(response.ok) def test_insert_cultivars(self): - csv_filename = 'cultivars.csv' - csv = 'input_files/'+csv_filename - files = {'fileName': open(csv, 'rb')} - response = requests.post( - 'http://0.0.0.0:6001/cultivars', files=files) + csv_filename = "cultivars.csv" + csv = "input_files/" + csv_filename + files = {"fileName": open(csv, "rb")} + response = requests.post("http://0.0.0.0:6001/cultivars", files=files) self.assertEqual(response.status_code, 200) self.assertTrue(response.ok) def test_insert_experimentSites(self): - csv_filename = 'experiments_sites.csv' - csv = 'input_files/'+csv_filename - files = {'fileName': open(csv, 'rb')} - response = requests.post( - 'http://0.0.0.0:6001/experiments_sites', files=files) + csv_filename = "experiments_sites.csv" + csv = "input_files/" + csv_filename + files = {"fileName": open(csv, "rb")} + response = requests.post("http://0.0.0.0:6001/experiments_sites", files=files) self.assertEqual(response.status_code, 200) self.assertTrue(response.ok) def test_insert_experimentTreatments(self): - csv_filename = 'experiments_treatments.csv' - csv = 'input_files/'+csv_filename - files = {'fileName': open(csv, 'rb')} + csv_filename = "experiments_treatments.csv" + csv = "input_files/" + csv_filename + files = {"fileName": open(csv, "rb")} response = requests.post( - 'http://0.0.0.0:6001/experiments_treatments', files=files) + "http://0.0.0.0:6001/experiments_treatments", files=files + ) self.assertEqual(response.status_code, 200) self.assertTrue(response.ok) def test_insert_sitesCultivars(self): - csv_filename = 'sites_cultivars.csv' - csv = 'input_files/'+csv_filename - files = {'fileName': open(csv, 'rb')} - response = requests.post( - 'http://0.0.0.0:6001/sites_cultivars', files=files) + csv_filename = "sites_cultivars.csv" + csv = "input_files/" + csv_filename + files = {"fileName": open(csv, "rb")} + response = requests.post("http://0.0.0.0:6001/sites_cultivars", files=files) self.assertEqual(response.status_code, 200) self.assertTrue(response.ok) def test_insert_citationsSites(self): - csv_filename = 'citations_sites.csv' - csv = 'input_files/'+csv_filename - files = {'fileName': open(csv, 'rb')} - response = requests.post( - 'http://0.0.0.0:6001/citations_sites', files=files) + csv_filename = "citations_sites.csv" + csv = "input_files/" + csv_filename + files = {"fileName": open(csv, "rb")} + response = requests.post("http://0.0.0.0:6001/citations_sites", files=files) self.assertEqual(response.status_code, 200) self.assertTrue(response.ok) diff --git a/yaba_test/test_yaba.py b/yaba_test/test_yaba.py index 9eb2d19..6f24123 100644 --- a/yaba_test/test_yaba.py +++ b/yaba_test/test_yaba.py @@ -4,100 +4,112 @@ import time -BASE_URL = 'http://0.0.0.0:5001/' +BASE_URL = "http://0.0.0.0:5001/" class BasicTests(unittest.TestCase): - def test_insert_experiments(self): - csv_filename = 'experiments.csv' - csv_path = 'input_files/experiments.csv' - files = {'fileName': open(csv_path, 'rb')} + csv_filename = "experiments.csv" + csv_path = "input_files/experiments.csv" + files = {"fileName": open(csv_path, "rb")} response = requests.post( - 'http://0.0.0.0:5001/yaba/v1/experiments?username=guestuser', files=files) + "http://0.0.0.0:5001/yaba/v1/experiments?username=guestuser", files=files + ) self.assertEqual(response.status_code, 201) self.assertTrue(response.ok) def test_insert_sites(self): - csv_filename = 'sites.csv' - csv_path = 'input_files/'+csv_filename - - shp_file = 'input_files/S8_two_row_polys.shp' - dbf_file = 'input_files/S8_two_row_polys.dbf' - prj_file = 'input_files/S8_two_row_polys.prj' - shx_file = 'input_files/S8_two_row_polys.shx' - - files= {'fileName': open(csv_path, 'rb'), - 'shp_file': open(shp_file, 'rb'), - 'dbf_file':open(dbf_file, 'rb'), - 'prj_file': open(prj_file, 'rb'), - 'shx_file': open(shx_file, 'rb')} - - response = requests.post('http://0.0.0.0:5001/yaba/v1/sites', - files=files) + csv_filename = "sites.csv" + csv_path = "input_files/" + csv_filename + + shp_file = "input_files/S8_two_row_polys.shp" + dbf_file = "input_files/S8_two_row_polys.dbf" + prj_file = "input_files/S8_two_row_polys.prj" + shx_file = "input_files/S8_two_row_polys.shx" + + files = { + "fileName": open(csv_path, "rb"), + "shp_file": open(shp_file, "rb"), + "dbf_file": open(dbf_file, "rb"), + "prj_file": open(prj_file, "rb"), + "shx_file": open(shx_file, "rb"), + } + + response = requests.post("http://0.0.0.0:5001/yaba/v1/sites", files=files) self.assertEqual(response.status_code, 201) self.assertTrue(response.ok) def test_insert_treatments(self): - csv_filename = 'treatments.csv' - csv = 'input_files/'+csv_filename - files = {'fileName': open(csv, 'rb')} + csv_filename = "treatments.csv" + csv = "input_files/" + csv_filename + files = {"fileName": open(csv, "rb")} response = requests.post( - 'http://0.0.0.0:5001/yaba/v1/treatments?username=guestuser', files=files) + "http://0.0.0.0:5001/yaba/v1/treatments?username=guestuser", files=files + ) self.assertEqual(response.status_code, 201) self.assertTrue(response.ok) def test_insert_cultivars(self): - csv_filename = 'cultivars.csv' - csv = 'input_files/'+csv_filename - files = {'fileName': open(csv, 'rb')} + csv_filename = "cultivars.csv" + csv = "input_files/" + csv_filename + files = {"fileName": open(csv, "rb")} response = requests.post( - 'http://0.0.0.0:5001/yaba/v1/cultivars?username=guestuser', files=files) + "http://0.0.0.0:5001/yaba/v1/cultivars?username=guestuser", files=files + ) self.assertEqual(response.status_code, 201) self.assertTrue(response.ok) def test_insert_citations(self): - csv_filename = 'citations.csv' - csv = 'input_files/'+csv_filename - files = {'fileName': open(csv, 'rb')} + csv_filename = "citations.csv" + csv = "input_files/" + csv_filename + files = {"fileName": open(csv, "rb")} response = requests.post( - 'http://0.0.0.0:5001/yaba/v1/citations?username=guestuser', files=files) + "http://0.0.0.0:5001/yaba/v1/citations?username=guestuser", files=files + ) self.assertEqual(response.status_code, 201) self.assertTrue(response.ok) - + def test_insert_experimentSites(self): - csv_filename = 'experiments_sites.csv' - csv = 'input_files/'+csv_filename - files = {'fileName': open(csv, 'rb')} + csv_filename = "experiments_sites.csv" + csv = "input_files/" + csv_filename + files = {"fileName": open(csv, "rb")} response = requests.post( - 'http://0.0.0.0:5001/yaba/v1/experiments_sites?username=guestuser', files=files) + "http://0.0.0.0:5001/yaba/v1/experiments_sites?username=guestuser", + files=files, + ) self.assertEqual(response.status_code, 201) self.assertTrue(response.ok) def test_insert_experimentTreatments(self): - csv_filename = 'experiments_treatments.csv' - csv = 'input_files/'+csv_filename - files = {'fileName': open(csv, 'rb')} + csv_filename = "experiments_treatments.csv" + csv = "input_files/" + csv_filename + files = {"fileName": open(csv, "rb")} response = requests.post( - 'http://0.0.0.0:5001/yaba/v1/experiments_treatments?username=guestuser', files=files) + "http://0.0.0.0:5001/yaba/v1/experiments_treatments?username=guestuser", + files=files, + ) self.assertEqual(response.status_code, 201) self.assertTrue(response.ok) def test_insert_sitesCultivars(self): - csv_filename = 'sites_cultivars.csv' - csv = 'input_files/'+csv_filename - files = {'fileName': open(csv, 'rb')} + csv_filename = "sites_cultivars.csv" + csv = "input_files/" + csv_filename + files = {"fileName": open(csv, "rb")} response = requests.post( - 'http://0.0.0.0:5001/yaba/v1/sites_cultivars?username=guestuser', files=files) + "http://0.0.0.0:5001/yaba/v1/sites_cultivars?username=guestuser", + files=files, + ) self.assertEqual(response.status_code, 201) self.assertTrue(response.ok) def test_insert_citationsSites(self): - csv_filename = 'citations_sites.csv' - csv = 'input_files/'+csv_filename - files = {'fileName': open(csv, 'rb')} + csv_filename = "citations_sites.csv" + csv = "input_files/" + csv_filename + files = {"fileName": open(csv, "rb")} response = requests.post( - 'http://0.0.0.0:5001/yaba/v1/citations_sites?username=guestuser', files=files) + "http://0.0.0.0:5001/yaba/v1/citations_sites?username=guestuser", + files=files, + ) self.assertEqual(response.status_code, 201) self.assertTrue(response.ok)