Skip to content

Commit

Permalink
fix sheet
Browse files Browse the repository at this point in the history
  • Loading branch information
@super622
super622 committed Sep 26, 2023
1 parent e5d2b68 commit 491fb50
Show file tree
Hide file tree
Showing 3 changed files with 165 additions and 37 deletions.
189 changes: 155 additions & 34 deletions Chating.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ async def append_to_gifusers(gifs_users, res):
flag = False
for user in gifs_users:
if(user['UserName'] == res['UserName'] and user['GifType'] == res['GifType']):
user['Coin'] = int(res['Coin']) * int(user['Gif_Count'] + 1)
user['Gif_Count'] += 1
user['Coin'] = int(res['Coin']) + int(user['Coin'])
user['Gif_Count'] = int(res['Gif_Count']) + int(user['Gif_Count'])
flag = True

if flag:
Expand Down Expand Up @@ -124,14 +124,67 @@ async def get_sheet_by_name(file_name, folder_name):
else:
return items[0]['id']

# add all gif users
async def append_to_total_gif_users(total_users, sub_users):
flag = False
for sub in sub_users:
for total in total_users:
if(total['UserName'] == sub['UserName'] and total['GifType'] == sub['GifType']):
total['Coin'] = int(sub['Coin']) + int(total['Coin'])
total['Gif_Count'] = int(total['Gif_Count']) + int(sub['Gif_Count'])
flag = True

if(flag != True):
total_users.append(sub)
return total_users
else:
flag = False

# add all snack users
async def append_to_total_snack_users(total_users, sub_users):
flag = False
for sub in sub_users:
for total in total_users:
if(total['UserName'] == sub['UserName']):
total['Gif_Count'] = int(sub['Gif_Count']) + int(total['Gif_Count'])
total['Coin'] = int(sub['Coin']) + int(total['Coin'])
total['Snack_Count'] = int(sub['Snack_Count']) + int(total['Snack_Count'])
flag = True

if(flag):
flag = False
else:
total_users.append(sub)
return total_users

# add all result
async def append_to_total_result(gif_users, snack_users):
max_len = len(gif_users) if len(gif_users) > len(snack_users) else len(snack_users)
temp_arr = None

if max_len == len(gif_users):
temp_arr = gif_users
else:
temp_arr = snack_users

for i in range(len(temp_arr)):
res_arr = None
if(i >= len(gif_users)):
res_arr = ['', '', '', '', snack_users[i]['UserName'], snack_users[i]['Snack_Count'], snack_users[i]['Gif_Count'], snack_users[i]['Coin']]
elif(i >= len(snack_users)):
res_arr = [gif_users[i]['UserName'], gif_users[i]['GifType'], gif_users[i]['Gif_Count'], gif_users[i]['Coin'], '','','','']
elif(i < len(gif_users) and i < len(snack_users)):
res_arr = [gif_users[i]['UserName'], gif_users[i]['GifType'], gif_users[i]['Gif_Count'], gif_users[i]['Coin'], snack_users[i]['UserName'], snack_users[i]['Snack_Count'], snack_users[i]['Gif_Count'], snack_users[i]['Coin']]

total_results.append(res_arr)

# add git and snack user
async def append_to_snack_gifusers(snack_gifs_users, gifs_user, username, snack_cnt):
flag = False
x = 1
for user in snack_gifs_users:
if(user['UserName'] == username):
user['Gif_Count'] = gifs_user['Gif_Count']
user['Coin'] = ((int(user['Snack_Count']) + int(snack_cnt)) * x) + int(gifs_user['Coin'])
user['Coin'] = int(gifs_user['Coin'])
user['Snack_Count'] = int(snack_cnt) + int(user['Snack_Count'])
flag = True

Expand All @@ -142,7 +195,7 @@ async def append_to_snack_gifusers(snack_gifs_users, gifs_user, username, snack_
"UserName": username,
"Gif_Count": gifs_user['Gif_Count'],
"Snack_Count": int(snack_cnt),
"Coin": (int(snack_cnt) * x) + int(gifs_user['Coin'])
"Coin": int(gifs_user['Coin'])
}
snack_gifs_users.append(res)
return snack_gifs_users
Expand All @@ -168,7 +221,7 @@ async def append_to_snack_gifusers(snack_gifs_users, gifs_user, username, snack_

if len(live_stream_id_arr) == 0:
return 'failure - not exist live stream id'
print(live_stream_id_arr)

for live_room_id in live_stream_id_arr:
url = f'https://17.live/ja/live/{live_room_id}'

Expand All @@ -179,15 +232,14 @@ async def append_to_snack_gifusers(snack_gifs_users, gifs_user, username, snack_

browser = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options)

# # Maximize the browser window
# browser.maximize_window()

# # Bring the browser window to the front
# browser.execute_script("window.focus();")
# time.sleep(3)

browser.get(url)
time.sleep(10)
total_snack_cnt = 0
total_coin_cnt = 0
total_score = 0
total_gif_man_cnt = 0
total_gifs_user = []
total_snack_user = []
total_results = []

while True:
time.sleep(5)
Expand All @@ -200,7 +252,7 @@ async def append_to_snack_gifusers(snack_gifs_users, gifs_user, username, snack_
gifs_users = []
snack_gifs_users = []
gifs_list = []
total_result = []
sub_result = []

chating_elements = browser.find_elements('css selector', '.Chat__ChatWrapper-sc-clenhv-0')
for chat_element in chating_elements:
Expand Down Expand Up @@ -238,6 +290,9 @@ async def append_to_snack_gifusers(snack_gifs_users, gifs_user, username, snack_

gif_man_cnt = len(gifs_users)
snack_cnt = len(snack_gifs_users)
total_snack_cnt += snack_cnt

total_gif_man_cnt = gif_man_cnt

for user in gifs_users:
coin_cnt += int(user['Coin'])
Expand All @@ -264,40 +319,81 @@ async def append_to_snack_gifusers(snack_gifs_users, gifs_user, username, snack_
elif(i < len(gifs_users) and i < len(snack_gifs_users)):
res_arr = [gifs_users[i]['UserName'], gifs_users[i]['GifType'], gifs_users[i]['Gif_Count'], gifs_users[i]['Coin'], snack_gifs_users[i]['UserName'], snack_gifs_users[i]['Snack_Count'], snack_gifs_users[i]['Gif_Count'], snack_gifs_users[i]['Coin']]

total_result.append(res_arr)
i += 1
sub_result.append(res_arr)

score_elements = browser.find_elements(By.XPATH, "//*[@style='transform: rotateX(0deg) translateZ(28px);']")
for element in score_elements:
while element.text == '':
time.sleep(2)
score += element.text
print(f"coin = {coin_cnt}, score = {score}")
time.sleep(5)

total_coin_cnt += coin_cnt
total_score = score

# total_gifs_user
await append_to_total_gif_users(total_gifs_user, gifs_users)

# total_snack_user
await append_to_total_snack_users(total_snack_user, snack_gifs_users)

# total result
await append_to_total_result(total_gifs_user, total_snack_user)

print(total_snack_cnt, total_coin_cnt, total_gif_man_cnt, total_score)
print(total_gifs_user)
print(total_snack_user)
print(total_results)

# create google sheet
filename = f"Test_{self.name}__{self.start_year}_{self.start_month}_{self.start_day}"
SCOPES = ['https://www.googleapis.com/auth/drive']
SERVICE_ACCOUNT_FILE = 'service-account.json'

creds = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE, scopes=SCOPES)
client = gspread.authorize(creds)
service = build('sheets', 'v4', credentials=creds)

tab_position = 0
filename = f"{self.name}"
folder_name = '1IkovXnPZ8y-aIgR6MnbykOVfXC34CJhT'
sheetID = await get_sheet_by_name(filename, folder_name)
print(sheetID)
create_flag = False
if(sheetID == ''):
create_flag = True
sheetID = await createGoogleSheet(filename)
print(sheetID)
tab_position = 3
else:
spreadsheet = client.open_by_key(sheetID)
sheets_list = spreadsheet.worksheets()
tab_position = len(sheets_list)

# write content into google sheet (init column name)
SCOPES = ['https://www.googleapis.com/auth/drive']
SERVICE_ACCOUNT_FILE = 'service-account.json'

creds = service_account.Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE, scopes=SCOPES)
client = gspread.authorize(creds)
service = build('sheets', 'v4', credentials=creds)

spreadsheet = client.open_by_key(sheetID)

current_month = datetime.datetime.now(pytz.timezone('Asia/Tokyo')).month
current_day = datetime.datetime.now(pytz.timezone('Asia/Tokyo')).day

if(create_flag):
worksheet = spreadsheet.sheet1
worksheet.update_title('チャット履歴')

worksheet.update_title(f"{current_month}-{current_day}")

worksheet.update("E1", [["コイン数"]], value_input_option="USER_ENTERED")
worksheet.update("E2", [["ギフト人数"]], value_input_option="USER_ENTERED")
worksheet.update("E3", [["スナック数"]], value_input_option="USER_ENTERED")
worksheet.update("E4", [["スコア"]], value_input_option="USER_ENTERED")

worksheet.update("A5", [["リスナー名"]], value_input_option="USER_ENTERED")
worksheet.update("B5", [["ギフト名"]], value_input_option="USER_ENTERED")
worksheet.update("C5", [["ギフト個数"]], value_input_option="USER_ENTERED")
worksheet.update("D5", [["コイン数"]], value_input_option="USER_ENTERED")
worksheet.update("E5", [["リスナー名"]], value_input_option="USER_ENTERED")
worksheet.update("F5", [["スナック"]], value_input_option="USER_ENTERED")
worksheet.update("G5", [["ギフト個数"]], value_input_option="USER_ENTERED")
worksheet.update("H5", [["合計コイン"]], value_input_option="USER_ENTERED")

worksheet = spreadsheet.add_worksheet(title="total", rows='5000', cols='8')

worksheet.update("E1", [["コイン数"]], value_input_option="USER_ENTERED")
worksheet.update("E2", [["ギフト人数"]], value_input_option="USER_ENTERED")
Expand All @@ -313,35 +409,60 @@ async def append_to_snack_gifusers(snack_gifs_users, gifs_user, username, snack_
worksheet.update("G5", [["ギフト個数"]], value_input_option="USER_ENTERED")
worksheet.update("H5", [["合計コイン"]], value_input_option="USER_ENTERED")

worksheet = spreadsheet.add_worksheet(title="ギフト内訳", rows='1000', cols='20')
worksheet = spreadsheet.add_worksheet(title="ギフト内訳", rows='5000', cols='3')

worksheet.update("A1", [["ギフト名"]], value_input_option="USER_ENTERED")
worksheet.update("B1", [["ギフト個数"]], value_input_option="USER_ENTERED")
worksheet.update("C1", [["コイン数"]], value_input_option="USER_ENTERED")

print(tab_position - 3)
worksheet = spreadsheet.get_worksheet(tab_position - 3)
if(worksheet == None):
worksheet = spreadsheet.add_worksheet(title=f"{current_month}-{current_day}", rows='5000', cols='8')

worksheet.update("E1", [["コイン数"]], value_input_option="USER_ENTERED")
worksheet.update("E2", [["ギフト人数"]], value_input_option="USER_ENTERED")
worksheet.update("E3", [["スナック数"]], value_input_option="USER_ENTERED")
worksheet.update("E4", [["スコア"]], value_input_option="USER_ENTERED")

worksheet.update("A5", [["リスナー名"]], value_input_option="USER_ENTERED")
worksheet.update("B5", [["ギフト名"]], value_input_option="USER_ENTERED")
worksheet.update("C5", [["ギフト個数"]], value_input_option="USER_ENTERED")
worksheet.update("D5", [["コイン数"]], value_input_option="USER_ENTERED")
worksheet.update("E5", [["リスナー名"]], value_input_option="USER_ENTERED")
worksheet.update("F5", [["スナック"]], value_input_option="USER_ENTERED")
worksheet.update("G5", [["ギフト個数"]], value_input_option="USER_ENTERED")
worksheet.update("H5", [["合計コイン"]], value_input_option="USER_ENTERED")

# clear content in google sheet
sheet_range = f'チャット履歴!A6:Z' # Adjust the range as needed
sheet_range = f'{current_month}-{current_day}!A6:Z' # Adjust the range as needed
service.spreadsheets().values().clear(spreadsheetId=sheetID, range=sheet_range).execute()
sheet_range = f'total!A6:Z'
service.spreadsheets().values().clear(spreadsheetId=sheetID, range=sheet_range).execute()
sheet_range = f'ギフト内訳!A2:Z' # Adjust the range as needed
service.spreadsheets().values().clear(spreadsheetId=sheetID, range=sheet_range).execute()

# write content into google sheet
worksheet = spreadsheet.worksheet("チャット履歴")
worksheet = spreadsheet.get_worksheet(tab_position - 3)

worksheet.update("F1", [[str(coin_cnt)]], value_input_option="USER_ENTERED")
worksheet.update("F2", [[str(snack_cnt)]], value_input_option="USER_ENTERED")
worksheet.update("F3", [[str(gif_man_cnt)]], value_input_option="USER_ENTERED")
worksheet.update("F4", [[str(score)]], value_input_option="USER_ENTERED")
try:
worksheet.insert_rows(total_result, row=6)
worksheet.insert_rows(sub_result, row=6)
except:
print('quota <')

worksheet = spreadsheet.worksheet("total")
try:
worksheet.insert_rows(total_results, row=6)
except:
print('quota <')

worksheet = spreadsheet.worksheet("ギフト内訳")
worksheet.insert_rows(gifs_list, row=2)

current_month = datetime.datetime.now(pytz.timezone('Asia/Tokyo')).month
current_day = datetime.datetime.now(pytz.timezone('Asia/Tokyo')).day
current_hour = datetime.datetime.now(pytz.timezone('Asia/Tokyo')).hour
current_minute = datetime.datetime.now(pytz.timezone('Asia/Tokyo')).minute

Expand Down
12 changes: 9 additions & 3 deletions app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import datetime
from multiprocessing import Process
import threading
import time
import schedule
import pytz

Expand All @@ -29,6 +30,7 @@ def change_string(value):
return value

def chating_scraping(end_date_month, end_date_day, end_time_hour, end_time_minute, nick_url):
print('start')
getChatingData = Chating(end_date_month, end_date_day, end_time_hour, end_time_minute, nick_url)
response = asyncio.run(getChatingData.main())
return json.dumps(response)
Expand All @@ -41,7 +43,7 @@ def event_scraping(start_date_month, start_date_day, event_url):

@app.route('/start', methods=['POST'])
def start():
url_type = bool(request.values.get('type'))
url_type = request.values.get('type')
start_date_year = int(request.values.get('start_date_year'))
start_date_month = int(request.values.get('start_date_month'))
start_date_day = int(request.values.get('start_date_day'))
Expand All @@ -65,14 +67,14 @@ def start():
end_datetime = japan_timezone.localize(datetime.datetime(end_date_year, end_date_month, end_date_day, end_time_hour, end_time_minute, 0))

# Convert current time to Japan time zone and make it offset-aware
cur_time = datetime.datetime.now(japan_timezone)
cur_time = datetime.datetime.now(pytz.timezone('Asia/Tokyo'))

# Calculate the delay in seconds until the scheduled time
delay = (start_datetime - cur_time).total_seconds()
delay = math.floor(delay)
print(delay)

if(url_type):
if(url_type != 'false'):
if(purpose_url.find(';') > -1):
event_url_arr = purpose_url.split(';')
for event in event_url_arr:
Expand All @@ -83,12 +85,16 @@ def start():
job = schedule.every().day.at(f"{change_string(start_time_hour)}:{change_string(start_time_minute)}", "Asia/Tokyo").do(event_scraping, start_date_month, start_date_day, purpose_url)
print(job)
scheduled_jobs[job] = {'start_datetime': start_datetime, 'end_datetime': end_datetime}
while True:
schedule.run_pending()
time.sleep(1000)
else:
if(purpose_url.find(';') > -1):
nick_name_arr = purpose_url.split(';')
for nick_name in nick_name_arr:
res = threading.Timer(delay, chating_scraping, args=(end_date_month, end_date_day, end_time_hour, end_time_minute, nick_name)).start()
else:
print(purpose_url)
res = threading.Timer(delay, chating_scraping, args=(end_date_month, end_date_day, end_time_hour, end_time_minute, purpose_url)).start()

# Convert start_date and end_date to datetime objects
Expand Down
1 change: 1 addition & 0 deletions getContents.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ def __init__(self, month, day, event_url):
self.month = month
self.day = day
self.event_url = event_url
self.date_str = ''

# Get Data from purpose site
async def scanData(self):
Expand Down

0 comments on commit 491fb50

Please sign in to comment.