diff --git a/api/crud/phone_data.py b/api/crud/phone_data.py index cc36d65..d5a7009 100644 --- a/api/crud/phone_data.py +++ b/api/crud/phone_data.py @@ -64,6 +64,16 @@ def merge_phonescraper_data(phonescraper_data: models.PhoneScraper, db: Session db.commit() db.close() + +def merge_phonescraper_data_list(scraper_list: List[models.PhoneScraper], db: Session = SessionLocal()): + """update models.Phone with list of models""" + + for scraped_data in scraper_list: + db.merge(scraped_data) + + db.commit() + db.close() + # Job Status def startjob(jobname: str, db: Session = SessionLocal()): diff --git a/api/routes/phone_data.py b/api/routes/phone_data.py index 5cdcb87..7ab6144 100644 --- a/api/routes/phone_data.py +++ b/api/routes/phone_data.py @@ -94,10 +94,13 @@ async def get_initiate_phone_scrape_now(*, token: str = Security(is_auth), backg response_model=List[schemas.PhoneInfo], ) def get_phone_info(*, token: str = Security(is_auth)): + logger.debug(f"DB query start at {datetime.datetime.now()}") results = crud.get_all_phone_data() + logger.debug(f"DB query end at {datetime.datetime.now()}") list_phone_schema_obj = [] + logger.debug(f"Schema creation start at {datetime.datetime.now()}") for item in results: phone_schema_obj = schemas.PhoneInfo( @@ -120,6 +123,8 @@ def get_phone_info(*, token: str = Security(is_auth)): ) list_phone_schema_obj.append(phone_schema_obj) + + logger.debug(f"Schema creation end at {datetime.datetime.now()}") return list_phone_schema_obj @@ -194,4 +199,101 @@ def get_phone_scraper_info(*, token: str = Security(is_auth)): return list_phone_scraper_schema_obj +# Phone Info & Scrape Data - called by VueJS to display all phone data (CUCM API and Phonescraper) combined page + +# The schema version takes 38 seconds to return the HTTP data for 30k records in phone info and 9k records in scraper. this is too long.. +@router.get( + '/allschema', + summary="Displays all phone data (CUCM API + Scraper)", + description="Returns list of phone data", + response_model=List[schemas.Phone_Cucm_Scraper_Combined] + ) +def get_phone_all_schema(*, token: str = Security(is_auth)): + logger.debug(f"DB query start at {datetime.datetime.now()}") + results = crud.get_all_scraper_data() + logger.debug(f"DB query end at {datetime.datetime.now()}") + + logger.debug(f"Schema creation start at {datetime.datetime.now()}") + list_phone_cucm_scraper_combined_schema_obj = [] + + for item in results: + phone_cucm_scraper_combined_schema_obj = schemas.Phone_Cucm_Scraper_Combined( + dname = item.devicename, + fw = item.firmware, + ipv4 = item.ipv4, + fdate = "" if item.first_seen_reg is None else item.first_seen_reg.strftime("%m/%d/%y %H:%M:%S"), + ldate = "" if item.last_seen_reg is None else item.last_seen_reg.strftime("%m/%d/%y %H:%M:%S") , + regstamp = "" if item.registration_time is None else item.registration_time.strftime("%m/%d/%y %H:%M:%S"), + cluster = item.cluster, + prot = item.Protocol, + model = item.Model.replace("Cisco ",""), + + # AXL Fields + dpool = item.devicepool, + dcss = item.devicecss, + descr = item.description, + em_profile = item.em_profile, + em_time = "" if item.em_time is None else item.em_time.strftime("%m/%d/%y %H:%M:%S"), + ) + + if item.phonescrape != None: + phone_cucm_scraper_combined_schema_obj.sn = item.phonescrape.sn + phone_cucm_scraper_combined_schema_obj.firmware = item.phonescrape.firmware + phone_cucm_scraper_combined_schema_obj.dn = item.phonescrape.dn + phone_cucm_scraper_combined_schema_obj.model = item.phonescrape.model + phone_cucm_scraper_combined_schema_obj.kem1 = item.phonescrape.kem1 + phone_cucm_scraper_combined_schema_obj.kem2 = item.phonescrape.kem2 + phone_cucm_scraper_combined_schema_obj.domain_name = item.phonescrape.domain_name + phone_cucm_scraper_combined_schema_obj.dhcp_server = item.phonescrape.dhcp_server + phone_cucm_scraper_combined_schema_obj.dhcp = item.phonescrape.dhcp + phone_cucm_scraper_combined_schema_obj.ip_address = item.phonescrape.ip_address + phone_cucm_scraper_combined_schema_obj.subnetmask = item.phonescrape.subnetmask + phone_cucm_scraper_combined_schema_obj.gateway = item.phonescrape.gateway + phone_cucm_scraper_combined_schema_obj.dns1 = item.phonescrape.dns1 + phone_cucm_scraper_combined_schema_obj.dns2 = item.phonescrape.dns2 + phone_cucm_scraper_combined_schema_obj.alt_tftp = item.phonescrape.alt_tftp + phone_cucm_scraper_combined_schema_obj.tftp1 = item.phonescrape.tftp1 + phone_cucm_scraper_combined_schema_obj.tftp2 = item.phonescrape.tftp2 + phone_cucm_scraper_combined_schema_obj.op_vlan = item.phonescrape.op_vlan + phone_cucm_scraper_combined_schema_obj.admin_vlan = item.phonescrape.admin_vlan + phone_cucm_scraper_combined_schema_obj.cucm1 = item.phonescrape.cucm1 + phone_cucm_scraper_combined_schema_obj.cucm2 = item.phonescrape.cucm2 + phone_cucm_scraper_combined_schema_obj.cucm3 = item.phonescrape.cucm3 + phone_cucm_scraper_combined_schema_obj.cucm4 = item.phonescrape.cucm4 + phone_cucm_scraper_combined_schema_obj.cucm5 = item.phonescrape.cucm5 + phone_cucm_scraper_combined_schema_obj.info_url = item.phonescrape.info_url + phone_cucm_scraper_combined_schema_obj.dir_url = item.phonescrape.dir_url + phone_cucm_scraper_combined_schema_obj.msg_url = item.phonescrape.msg_url + phone_cucm_scraper_combined_schema_obj.svc_url = item.phonescrape.svc_url + phone_cucm_scraper_combined_schema_obj.idle_url = item.phonescrape.idle_url + phone_cucm_scraper_combined_schema_obj.info_url_time = item.phonescrape.info_url_time + phone_cucm_scraper_combined_schema_obj.proxy_url = item.phonescrape.proxy_url + phone_cucm_scraper_combined_schema_obj.auth_url = item.phonescrape.auth_url + phone_cucm_scraper_combined_schema_obj.tvs = item.phonescrape.tvs + phone_cucm_scraper_combined_schema_obj.CDP_Neighbor_ID = item.phonescrape.CDP_Neighbor_ID + phone_cucm_scraper_combined_schema_obj.CDP_Neighbor_IP = item.phonescrape.CDP_Neighbor_IP + phone_cucm_scraper_combined_schema_obj.CDP_Neighbor_Port = item.phonescrape.CDP_Neighbor_Port + phone_cucm_scraper_combined_schema_obj.LLDP_Neighbor_ID = item.phonescrape.LLDP_Neighbor_ID + phone_cucm_scraper_combined_schema_obj.LLDP_Neighbor_IP = item.phonescrape.LLDP_Neighbor_IP + phone_cucm_scraper_combined_schema_obj.LLDP_Neighbor_Port = item.phonescrape.LLDP_Neighbor_Port + phone_cucm_scraper_combined_schema_obj.ITL = item.phonescrape.ITL + phone_cucm_scraper_combined_schema_obj.last_scraped = "" if item.phonescrape.date_modified is None else item.phonescrape.date_modified.strftime("%m/%d/%y %H:%M:%S") + + list_phone_cucm_scraper_combined_schema_obj.append(phone_cucm_scraper_combined_schema_obj) + logger.debug(f"Schema creation end at {datetime.datetime.now()}") + + return list_phone_cucm_scraper_combined_schema_obj + + +@router.get( + '/alllist', + summary="Displays all phone data (CUCM API + Scraper)", + description="Returns list of phone data", + response_model=[] + ) +def get_phone_all_list(*, token: str = Security(is_auth)): + logger.debug(f"DB query start at {datetime.datetime.now()}") + results = crud.get_all_scraper_data() + logger.debug(f"DB query end at {datetime.datetime.now()}") + return results \ No newline at end of file diff --git a/api/schemas/phone_data.py b/api/schemas/phone_data.py index eb5182b..91e7711 100644 --- a/api/schemas/phone_data.py +++ b/api/schemas/phone_data.py @@ -69,4 +69,48 @@ class PhoneScraper(BaseModel): LLDP_Neighbor_IP: str = None LLDP_Neighbor_Port: str = None ITL: str = None - date_modified: str = None \ No newline at end of file + date_modified: str = None + + +class Phone_Cucm_Scraper_Combined(PhoneInfo): + sn: str = None + firmware: str = None + dn: str = None + model: str = None + kem1: str = None + kem2: str = None + domain_name: str = None + dhcp_server: str = None + dhcp: str = None + ip_address: str = None + subnetmask: str = None + gateway: str = None + dns1: str = None + dns2: str = None + alt_tftp: str = None + tftp1: str = None + tftp2: str = None + op_vlan: str = None + admin_vlan: str = None + cucm1: str = None + cucm2: str = None + cucm3: str = None + cucm4: str = None + cucm5: str = None + info_url: str = None + dir_url: str = None + msg_url: str = None + svc_url: str = None + idle_url: str = None + info_url_time: str = None + proxy_url: str = None + auth_url: str = None + tvs: str = None + CDP_Neighbor_ID: str = None + CDP_Neighbor_IP: str = None + CDP_Neighbor_Port: str = None + LLDP_Neighbor_ID: str = None + LLDP_Neighbor_IP: str = None + LLDP_Neighbor_Port: str = None + ITL: str = None + last_scraped: str = None \ No newline at end of file