asp_pool_hb.py

#!/usr/bin/env python
# coding=utf-8

import pprint
import csv
import click 
import requests
import datetime as datetime
from datetime import date
from xml.etree import ElementTree as ET
import os
import random
import json
# import logging
from requests.exceptions import ConnectionError
from requests.exceptions import ChunkedEncodingError
from requests.exceptions import ReadTimeout
# from multiprocessing.dummy import Pool as ThreadPool
import multiprocessing
from xml.etree.ElementTree import ParseError

from openpyxl import Workbook
from openpyxl import load_workbook

# HotelBeds Signature..
import time, hashlib
import copy

def validate_d(date_text):
	try:
		datetime.datetime.strptime(date_text, '%Y-%m-%d')
	except ValueError:
		raise ValueError("Incorrect data format, should be YYYY-MM-DD")

def daterange(start_date, end_date):
    for n in range(int ((end_date - start_date).days)):
        yield start_date + datetime.timedelta(n)

MAX_RETRIES = 2

def has_item_price(r):
	r_tree = ET.fromstring(r.text)
	if r_tree.find('.//ItemPrice') == None:
		return False
	return True

URL = 'https://api.hotelbeds.com/hotel-api/1.0/hotels'

def asp(s_request):
	r = None
	for i in range(MAX_RETRIES):
		# Signature is generated by SHA256 (Api-Key + Secret + Timestamp (in seconds))
		# sigStr = "%s%s%d" % (agent_secret['API-key'], agent_secret['Secret'], int(time.time()))
		sigStr = "%s%s%d" % (s_request['API-key'], s_request['Secret'], int(time.time()))
		signature = hashlib.sha256(sigStr.encode('utf-8')).hexdigest()
		# print(request_json)

		try:
			headers = { 'X-Signature': signature,
						'API-Key': s_request['API-key'],
						'Content-Type': 'application/json' }
			r = requests.post(URL, data=json.dumps(s_request['data']), headers=headers, timeout=10)
			print('PPID: {} PID: {} HB_key: {} tries: {}'.format(os.getppid(), os.getpid(), str(s_request['hotels']), str(i)) )
		except OSError:
			print('Error: ignoring OSError...' + str(i))
			continue
		except ConnectionError:
			print('Error: ignoring ConnectionError...' + str(i))
			continue
		except ChunkedEncodingError:
			print('Error: ignoring ChunkedEncodingError...' + str(i))
			continue
		except ReadTimeout:
			print('Error: ignoring ReadTimeout...' + str(i))
			continue
		else:
			break
		# try:
		# 	r = requests.post(url, data=s_request['body'], timeout=10)
		# 	# retry if no price found?
		# 	# if not has_item_price(r):
		# 	# 	print('retry.. ' + str(i))
		# 	# 	continue
		# 	# print('request body: ' + str(request_body[301:322]))
		# 	# print('Posting ' + str(s_request['GTA_key']))
		# 	print('PPID: {} PID: {} GTA_key: {}'.format(os.getppid(), os.getpid(), str(s_request['GTA_key'])) )
		# except OSError:
		# 	print('Error: ignoring OSError...' + str(i))
		# 	continue
		# except ConnectionError:
		# 	print('Error: ignoring ConnectionError...' + str(i))
		# 	continue
		# except ChunkedEncodingError:
		# 	print('Error: ignoring ChunkedEncodingError...' + str(i))
		# 	continue
		# except ReadTimeout:
		# 	print('Error: ignoring ReadTimeout...' + str(i))
		# 	continue
		# else:
		# 	break
	if r == None:
		print('Warning: Reached MAX RETRIES.. r==None.. ')

	s_request['response'] = r
	return s_request

def asp_p(search_requests):
	# pool = ThreadPool(threads)
	# results = pool.map(asp, search_requests)
	# pool.close()
	# pool.join()
	PROCESSES = 2
	with multiprocessing.Pool(PROCESSES) as pool:
		results = pool.map(asp, search_requests)
	return results

# def process(responese):
# 	if response == None:
# 		return None
# 	r_tree = ET.fromstring(response.text)
# 	if r_tree.find('.//ItemPrice') == None:
# 		return None
# 	for hotel in r_tree.find('.//HotelDetails'):
# 		hotel_name = hotel.find('.//Item').text
# 		for room_cat in r_tree.find('.//RoomCategories'):
# 			print('Id: ' + str(room_cat.get('Id')))
# 			print('Des: ' + str(room_cat.find('.//Description').text))
# 			entry = dict()
# 			entry['GTA_key'] = hotel_code['city_code'] + '_' + hotel_code['item_code']
# 			entry['Hotel_Name'] = hotel_name
# 			entry['Room_Name'] = room_cat.find('.//Description').text
# 			entry['Category_id'] = room_cat.get('Id')
# 			entry['Breakfast'] = room_cat.find('.//Basis').get('Code')
# 			entry['Policy'] = ''
# 			for charge_condition in room_cat.find('.//ChargeConditions'):
# 				if charge_condition.get('Type') == 'cancellation':
# 					for conditoin in charge_condition:
# 						if conditoin.get('Charge') == 'true':
# 							entry['Policy'] += 'Charge(FromDay: ' + str(conditoin.get('FromDay')) + ' ToDay: ' + str(conditoin.get('ToDay')) + ') '
# 						else:
# 							entry['Policy'] += 'Free(FromDay: ' + str(conditoin.get('FromDay')) + ') '

# 			entry['Check_in'] = checkin_date.strftime('%Y-%m-%d')
# 			entry['Price'] = room_cat.find('.//ItemPrice').text
# 			entry['Currency'] = room_cat.find('.//ItemPrice').get('Currency')
# 			return entry
# 			# res.append(entry)

# def process_p(search_responses, threads=2):
# 	pool = ThreadPool(threads)
# 	results = pool.map(process, search_responses)
# 	pool.close()
# 	pool.join()
# 	return results

def add_empty_ent(response, checkin_date, res):
	ent = {}
	ent['GTA_key'] = response['gta_key']
	ent['Check_in'] = checkin_date.strftime('%Y-%m-%d')
	res.append(ent)

@click.command()
@click.option('--hotel_file', default='hb_hotels_thai.xlsx')
# @click.option('--check_file', default='hb_image_check.xlsx')
@click.option('--from_d', default='2018-08-10')
@click.option('--to_d', default='2018-08-16')
@click.option('--client', default='ali_hb')
def asp_pool_hb(hotel_file, from_d, to_d, client):

	BATCH_N = 1000

	# wb2 = load_workbook('test.xlsx')
	try:
		print('Try loading hotel file workbook..')
		wb = load_workbook(hotel_file)
		print('Loading workbook.. done..')
	except FileNotFoundError:
		print('File not found.. Specify file in --hotel_file option')
		return

	ws = wb.active
	dc = [] 
	hy = []
	si = []
	hotel_ids = []
	for i, t in enumerate(tuple(ws.rows)):
		if i == 0:
			continue
		# print('i: ' + str(i))
		# print(t[0].value)
		# print(t[9].value)
		# if t[0].value == None or t[9].value == None:
		# 	print('Warning: No hotel id or contract type..')
		# 	continue

		if t[0].value == None:
			print('Warning: No hotel id or contract type..')
			continue

		if t[9].value == 'Direct Hotel':
			dc.append(t[0].value)
		if t[9].value == 'External with contract':
			hy.append(t[0].value)
		if t[9].value == 'External Only':
			si.append(t[0].value)

		hotel_ids.append(t[0].value)
	# print(str(dc))
	# print(str(hy))
	# print(str(si))

	agent_secret = None
	try:
		with open(os.path.join('C:\\Users\\809452\\gta_swarm', 'secrets.json'), encoding='utf-8') as data_file:    
			agent_secret = (json.load(data_file))[client]
	except FileNotFoundError:
		print('No secret file.. Exit(1)')
		return

	request_ents = []

	try:
		with open('availability.json') as f:
			request_json = json.load(f)
	except FileNotFoundError:
		print('No avail json file.. Exit(1)')
		return

	# print([hotel_ids[i * BATCH_N:(i + 1) * BATCH_N] for i in range((len(hotel_ids) + BATCH_N - 1) // BATCH_N )])
	# return 

	from_date = datetime.datetime.strptime(from_d, '%Y-%m-%d').date()
	to_date = datetime.datetime.strptime(to_d, '%Y-%m-%d').date()
	for date in [from_date + datetime.timedelta(days=x) for x in range(0, int((to_date - from_date).days))]:
		for batch_ids in [hotel_ids[i * BATCH_N:(i + 1) * BATCH_N] for i in range((len(hotel_ids) + BATCH_N - 1) // BATCH_N )]:
			ent = {}
			ent['API-key'] = agent_secret['API-key']
			ent['Secret'] = agent_secret['Secret']
			ent['checkIn'] = date.strftime('%Y-%m-%d')
			ent['checkOut'] = (date + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
			ent['hotels'] = batch_ids
			ent['data'] = copy.deepcopy(request_json)
			ent['data']['stay']['checkIn'] = ent['checkIn']
			ent['data']['stay']['checkOut'] = ent['checkOut']
			ent['data']['hotels']['hotel'] = ent['hotels']
			request_ents.append(ent)
			# print(str(ent))

	results = asp_p(request_ents)

	# try:
	# 	print('Try loading check file workbook..')
	# 	wb = load_workbook(check_file)
	# 	print('Loading workbook.. done..')
	# except FileNotFoundError:
	# 	print('File not found.. Specify file in --check_file option')
	# 	return

	# ws = wb.active
	# hotel_ids = [] 
	# for i, t in enumerate(tuple(ws.rows)):
	# 	if i == 0:
	# 		continue
	# 	# print('i: ' + str(i))
	# 	# print(t[0].value)
	# 	# print(t[9].value)
	# 	if t[15].value == None
	# 		print('Warning: No hotel id..')
	# 		continue

	# 	hotel_ids.append(t[15].value)

	# url = 'https://api.hotelbeds.com/hotel-api/1.0/hotels'
	# url_param = '?language=ENG&useSecondaryLanguage=False'
	# url_param_format = '?fields=all&language=ENG&from=1&to=1000'

	

	

	# print(r.text)
	res = []
	for res_ent in results:
		try:
			response = json.loads(res_ent['response'].text)
		except JSONDecodeError:
			print('Waarning: decode error.. ')
			for hotel_id in res_ent['hotels']:
				ent = {}
				ent['checkIn'] = res_ent['checkIn']
				ent['checkOut'] = res_ent['checkOut']
				ent['hotel_code'] = hotel_id
				res.append(ent)
			continue

		try:
			temp_ids = set()
		except KeyError:
			print('Warning: Key error.. line 293..')
		for hotel in response['hotels']['hotels']:
			for room in hotel['rooms']:
				for ratekey in room['rates']:
					ent = {}
					ent['checkIn'] = response['hotels']['checkIn']
					ent['checkOut'] = response['hotels']['checkOut']
					ent['hotel_code'] = hotel['code']
					temp_ids.add(hotel['code'])
					ent['hotel_name'] = hotel['name']
					ent['room_name'] = room['name']
					ent['ratekey'] = ratekey['rateKey']
					ent['net'] = ratekey['net']
					ent['currency'] = hotel['currency']
					res.append(ent)
		if response['hotels']['total'] != BATCH_N:
			for hotel_id in res_ent['hotels']:
				if hotel_id not in temp_ids:
					ent = {}
					ent['checkIn'] = res_ent['checkIn']
					ent['checkOut'] = res_ent['checkOut']
					ent['hotel_code'] = hotel_id
					res.append(ent)




	# res = []
	# search_requests = []

	# try:
	# 	validate_d(checkin_d)
	# except ValueError:
	# 	print('Please input date in correct format..')
	# 	return
	# checkin_date = datetime.datetime.strptime(checkin_d, '%Y-%m-%d').date()
	# print('Check in date ' + checkin_date.strftime('%Y-%m-%d'))

	# hotel_ids = set()
	# hotel_codes = []
	# with open(file_name, 'r') as file:
	# 	for line in file:
	# 		# pp.pprint(line)
	# 		if line in hotel_ids:
	# 			continue			
	# 		try:
	# 			city_code, item_code = line.rstrip().split('_')
	# 		except ValueError:
	# 			print('Warning: skipping GTA key.. ' + line.rstrip())
	# 			continue
	# 		hotel_codes.append(dict([('city_code', city_code), ('item_code', item_code)]))
	# 		hotel_ids.add(line)


	# search_tree = ET.parse(os.path.join(os.getcwd(), 'SearchHotelPricePaxRequest.xml'))
	# search_tree.find('.//RequestorID').set('Client', agent_secret['id'])
	# search_tree.find('.//RequestorID').set('EMailAddress', agent_secret['email'])
	# search_tree.find('.//RequestorID').set('Password', agent_secret['password'])

	# # prep search_requests
	# for counter, hotel_code in enumerate(hotel_codes):
	# 	msg = ' '.join([ 'Searching Price for',
	# 						hotel_code['city_code'],
	# 						hotel_code['item_code'],
	# 						'..',
	# 						'id:',
	# 						str(counter)
	# 					])
	# 	# print(msg)

	# 	search_tree.find('.//ItemDestination').set('DestinationCode', hotel_code['city_code'])
	# 	search_tree.find('.//ItemCode').text = hotel_code['item_code']
	# 	search_tree.find('.//PaxRoom').set('Adults', str(2))
	# 	search_tree.find('.//CheckInDate').text = checkin_date.strftime('%Y-%m-%d')
	# 	ent = {}
	# 	ent['GTA_key'] = '_'.join([hotel_code['city_code'], hotel_code['item_code']])
	# 	ent['body'] = ET.tostring(search_tree.getroot(), encoding='UTF-8', method='xml')
	# 	search_requests.append(ent)

	# # multi thread
	# responses = asp_p(search_requests)

	# # process res
	# for response in responses:
	# 	if response == None:
	# 		continue
	# 	try:
	# 		r_tree = ET.fromstring(response.text)
	# 	except ParseError:
	# 		print('Warning: Parse error for..\n' + response.text)
	# 		continue
	# 	if r_tree.find('.//ItemPrice') == None:
	# 		# add_empty_ent(response, checkin_date, res)
	# 		continue
	# 	for hotel in r_tree.find('.//HotelDetails'):
	# 		hotel_name = hotel.find('.//Item').text
	# 		gta_key = hotel.find('.//City').get('Code') + '_' + hotel.find('.//Item').get('Code')

	# 		for room_cat in r_tree.find('.//RoomCategories'):
	# 			# print('Id: ' + str(room_cat.get('Id')))
	# 			# print('Des: ' + str(room_cat.find('.//Description').text))
	# 			entry = dict()
	# 			# entry['GTA_key'] = hotel_code['city_code'] + '_' + hotel_code['item_code']
	# 			entry['GTA_key'] = gta_key
				
	# 			entry['Hotel_Name'] = hotel_name
	# 			entry['Room_Name'] = room_cat.find('.//Description').text
	# 			entry['Category_id'] = room_cat.get('Id')
	# 			entry['Breakfast'] = room_cat.find('.//Basis').get('Code')
	# 			entry['Policy'] = ''
	# 			for charge_condition in room_cat.find('.//ChargeConditions'):
	# 				if charge_condition.get('Type') == 'cancellation':
	# 					for conditoin in charge_condition:
	# 						if conditoin.get('Charge') == 'true':
	# 							entry['Policy'] += 'Charge(FromDay: ' + str(conditoin.get('FromDay')) + ' ToDay: ' + str(conditoin.get('ToDay')) + ') '
	# 						else:
	# 							entry['Policy'] += 'Free(FromDay: ' + str(conditoin.get('FromDay')) + ') '

	# 			entry['Check_in'] = checkin_date.strftime('%Y-%m-%d')
	# 			entry['Price'] = room_cat.find('.//ItemPrice').text
	# 			entry['Currency'] = room_cat.find('.//ItemPrice').get('Currency')
	# 			res.append(entry)

	# gta_keys = [ ent['GTA_key'] for ent in res ]
	# # print(str(gta_keys))
	# for hotel_id in hotel_ids:
	# 	if hotel_id != '' and hotel_id not in gta_keys:
	# 		# print(hotel_id)
	# 		ent = {}
	# 		ent['GTA_key'] = hotel_id.strip()
	# 		ent['Check_in'] = checkin_date.strftime('%Y-%m-%d')
	# 		res.append(ent)

	keys = None
	max_len = 0
	for ent in res:
		if len(ent.keys()) > max_len:
			max_len = len(ent.keys())
			keys = ent.keys()

	output_file_name = '_'.join([ 'Output_hb_search_price',
									client,
									# hotel_file,
									'from',
									from_date.strftime('%y%m%d'),
									# checkin_date.strftime('%y%m%d'),
									datetime.datetime.now().strftime('%H%M')
								]) + '.csv'
	# keys = res[0].keys()
	with open(output_file_name, 'w', newline='', encoding='utf-8') as output_file:
		dict_writer = csv.DictWriter(output_file, keys)
		dict_writer.writeheader()
		dict_writer.writerows(res)

if __name__ == '__main__':
	multiprocessing.freeze_support()
	asp_pool_hb()