147 lines
6.3 KiB
Python
147 lines
6.3 KiB
Python
import itertools
|
||
import re
|
||
import time
|
||
from collections import defaultdict
|
||
from PyQt5.QtCore import pyqtSignal, QObject
|
||
|
||
from unidecode import unidecode
|
||
|
||
import search
|
||
|
||
punctuation_to_none = str.maketrans({key: None for key in "!\"#$%&\'()*+,-.:;<=>?@[\\]^_`{|}~<7E>"})
|
||
punctuation_to_space = str.maketrans({key: " " for key in "!\"#$%&\'()*+,-.:;<=>?@[\\]^_`{|}~<7E>"})
|
||
|
||
class QuestionHandler(QObject):
|
||
searching = pyqtSignal([str, str, list, list])
|
||
googleResults = pyqtSignal([dict, bool])
|
||
pageResults = pyqtSignal([dict, bool])
|
||
|
||
async def answer_question(self, qn, question, original_answers):
|
||
print("Searching")
|
||
start = time.time()
|
||
|
||
question = unidecode(question)
|
||
|
||
answers = []
|
||
for ans in original_answers:
|
||
ans = unidecode(ans)
|
||
answers.append(ans.translate(punctuation_to_none))
|
||
#answers.append(ans.translate(punctuation_to_space))
|
||
answers = list(dict.fromkeys(answers))
|
||
print('Possible Answers: {}'.format(answers))
|
||
self.searching.emit(str(qn), question, original_answers, answers)
|
||
|
||
reverse = True
|
||
if ("NOT" in question and '"NOT"' not in question):
|
||
question = re.sub(r'\bNOT\b', ' ', question)
|
||
elif ("least" in question.lower() and "at least" not in question.lower() and '"least"' not in question.lower()):
|
||
question = re.sub(r'\bleast\b', ' most ', question)
|
||
elif ("fewest" in question and '"fewest"' not in question):
|
||
question = re.sub(r'\bfewest\b', ' most ', question)
|
||
elif ("NEVER" in question and '"NEVER"' not in question):
|
||
question = re.sub(r'\bNOT\b', ' always ', question)
|
||
elif ("NON" in question and '"NON"' not in question):
|
||
question = re.sub(r'\bNON', ' ', question)
|
||
else:
|
||
reverse = False
|
||
|
||
question_lower = question.lower().replace('"', '').replace("'", '"')
|
||
question_lower = question_lower.replace('which of these', 'what').replace('which', 'what')
|
||
|
||
"""quoted = re.findall('"([^"]*)"', question_lower) # Get all words in quotes
|
||
no_quote = question_lower
|
||
for quote in quoted:
|
||
no_quote = no_quote.replace(f"\"{quote}\"", "1placeholder1")
|
||
|
||
question_keywords = search.find_keywords(no_quote)
|
||
for quote in quoted:
|
||
question_keywords[question_keywords.index("1placeholder1")] = quote"""
|
||
|
||
#if 'which' in question_lower:
|
||
#question_lower += ' ({})'.format(' OR '.join(answers))
|
||
|
||
print('Search: "{}"'.format(question_lower))
|
||
smart_data, search_results = await search.search_google(question_lower, 5)
|
||
print('Smart Data: {}'.format(smart_data))
|
||
print('Search Results: {}'.format(search_results))
|
||
|
||
print('Searching smart Google data and website previews...')
|
||
search_text = [search.clean_html(x).lower().translate(punctuation_to_none) for x in smart_data]
|
||
counts = await self._search_method1(search_text, answers, reverse)
|
||
if counts:
|
||
self.googleResults.emit(counts, reverse)
|
||
else:
|
||
counts = await self._search_method2(search_text, answers, reverse)
|
||
print(counts)
|
||
if counts:
|
||
self.googleResults.emit(counts, reverse)
|
||
print('GOOGLE THINKS: {}'.format(min(counts, key=counts.get) if reverse else max(counts, key=counts.get)))
|
||
else:
|
||
self.googleResults.emit({k: 0 for k in answers}, False)
|
||
print('GOOGLE is not sure.')
|
||
|
||
|
||
print()
|
||
print('Searching full websites provided by Google...')
|
||
search_text = [x.translate(punctuation_to_none) for x in await search.get_clean_texts(search_results)]
|
||
counts = await self._search_method1(search_text, answers, reverse)
|
||
if counts:
|
||
self.pageResults.emit(counts, reverse)
|
||
else:
|
||
counts = await self._search_method2(search_text, answers, reverse)
|
||
if counts:
|
||
self.pageResults.emit(counts, reverse)
|
||
print('WEBSITES THINK: {}'.format(min(counts, key=counts.get) if reverse else max(counts, key=counts.get)))
|
||
else:
|
||
self.pageResults.emit({k: 0 for k in answers}, False)
|
||
print('WEBSITES are not sure.')
|
||
|
||
|
||
print(f"Search took {time.time() - start} seconds")
|
||
return ""
|
||
|
||
async def _search_method1(self, texts, answers, reverse):
|
||
"""
|
||
Returns the answer with the maximum/minimum number of exact occurrences in the texts.
|
||
:param texts: List of text to analyze
|
||
:param answers: List of answers
|
||
:param reverse: True if the best answer occurs the least, False otherwise
|
||
:return: Answer that occurs the most/least in the texts, empty string if there is a tie
|
||
"""
|
||
print("Running method 1")
|
||
counts = {answer.lower(): 0 for answer in answers}
|
||
|
||
for text in texts:
|
||
for answer in counts:
|
||
counts[answer] += len(re.findall(f"\\b{answer}\\b", text))
|
||
|
||
print(counts)
|
||
|
||
# If not all answers have count of 0 and the best value doesn't occur more than once, return the best answer
|
||
best_value = min(counts.values()) if reverse else max(counts.values())
|
||
if not all(c == 0 for c in counts.values()) and list(counts.values()).count(best_value) == 1:
|
||
return counts
|
||
return ""
|
||
|
||
async def _search_method2(self, texts, answers, reverse):
|
||
"""
|
||
Return the answer with the maximum/minimum number of keyword occurrences in the texts.
|
||
:param texts: List of text to analyze
|
||
:param answers: List of answers
|
||
:param reverse: True if the best answer occurs the least, False otherwise
|
||
:return: Answer whose keywords occur most/least in the texts
|
||
"""
|
||
print("Running method 2")
|
||
counts = {answer: {keyword: 0 for keyword in search.find_keywords(answer)} for answer in answers}
|
||
|
||
for text in texts:
|
||
for keyword_counts in counts.values():
|
||
for keyword in keyword_counts:
|
||
keyword_counts[keyword] += len(re.findall(f" {keyword} ", text))
|
||
|
||
print(counts)
|
||
counts_sum = {answer: sum(keyword_counts.values()) for answer, keyword_counts in counts.items()}
|
||
|
||
if not all(c == 0 for c in counts_sum.values()):
|
||
return counts_sum
|
||
return "" |