import itertools import re import time from collections import defaultdict from PyQt5.QtCore import pyqtSignal, QObject from unidecode import unidecode import search punctuation_to_none = str.maketrans({key: None for key in "!\"#$%&\'()*+,-.:;<=>?@[\\]^_`{|}~�"}) punctuation_to_space = str.maketrans({key: " " for key in "!\"#$%&\'()*+,-.:;<=>?@[\\]^_`{|}~�"}) class QuestionHandler(QObject): searching = pyqtSignal([str, str, list, list]) googleResults = pyqtSignal([dict, bool]) pageResults = pyqtSignal([dict, bool]) async def answer_question(self, qn, question, original_answers): print("Searching") start = time.time() question = unidecode(question) answers = [] for ans in original_answers: ans = unidecode(ans) answers.append(ans.translate(punctuation_to_none)) #answers.append(ans.translate(punctuation_to_space)) answers = list(dict.fromkeys(answers)) print('Possible Answers: {}'.format(answers)) self.searching.emit(str(qn), question, original_answers, answers) reverse = True if ("NOT" in question and '"NOT"' not in question): question = re.sub(r'\bNOT\b', ' ', question) elif ("least" in question.lower() and "at least" not in question.lower() and '"least"' not in question.lower()): question = re.sub(r'\bleast\b', ' most ', question) elif ("fewest" in question and '"fewest"' not in question): question = re.sub(r'\bfewest\b', ' most ', question) elif ("NEVER" in question and '"NEVER"' not in question): question = re.sub(r'\bNOT\b', ' always ', question) elif ("NON" in question and '"NON"' not in question): question = re.sub(r'\bNON', ' ', question) else: reverse = False question_lower = question.lower().replace('"', '').replace("'", '"') question_lower = question_lower.replace('which of these', 'what').replace('which', 'what') """quoted = re.findall('"([^"]*)"', question_lower) # Get all words in quotes no_quote = question_lower for quote in quoted: no_quote = no_quote.replace(f"\"{quote}\"", "1placeholder1") question_keywords = search.find_keywords(no_quote) for quote in quoted: question_keywords[question_keywords.index("1placeholder1")] = quote""" #if 'which' in question_lower: #question_lower += ' ({})'.format(' OR '.join(answers)) print('Search: "{}"'.format(question_lower)) smart_data, search_results = await search.search_google(question_lower, 5) print('Smart Data: {}'.format(smart_data)) print('Search Results: {}'.format(search_results)) print('Searching smart Google data and website previews...') search_text = [search.clean_html(x).lower().translate(punctuation_to_none) for x in smart_data] counts = await self._search_method1(search_text, answers, reverse) if counts: self.googleResults.emit(counts, reverse) else: counts = await self._search_method2(search_text, answers, reverse) print(counts) if counts: self.googleResults.emit(counts, reverse) print('GOOGLE THINKS: {}'.format(min(counts, key=counts.get) if reverse else max(counts, key=counts.get))) else: self.googleResults.emit({k: 0 for k in answers}, False) print('GOOGLE is not sure.') print() print('Searching full websites provided by Google...') search_text = [x.translate(punctuation_to_none) for x in await search.get_clean_texts(search_results)] counts = await self._search_method1(search_text, answers, reverse) if counts: self.pageResults.emit(counts, reverse) else: counts = await self._search_method2(search_text, answers, reverse) if counts: self.pageResults.emit(counts, reverse) print('WEBSITES THINK: {}'.format(min(counts, key=counts.get) if reverse else max(counts, key=counts.get))) else: self.pageResults.emit({k: 0 for k in answers}, False) print('WEBSITES are not sure.') print(f"Search took {time.time() - start} seconds") return "" async def _search_method1(self, texts, answers, reverse): """ Returns the answer with the maximum/minimum number of exact occurrences in the texts. :param texts: List of text to analyze :param answers: List of answers :param reverse: True if the best answer occurs the least, False otherwise :return: Answer that occurs the most/least in the texts, empty string if there is a tie """ print("Running method 1") counts = {answer.lower(): 0 for answer in answers} for text in texts: for answer in counts: counts[answer] += len(re.findall(f"\\b{answer}\\b", text)) print(counts) # If not all answers have count of 0 and the best value doesn't occur more than once, return the best answer best_value = min(counts.values()) if reverse else max(counts.values()) if not all(c == 0 for c in counts.values()) and list(counts.values()).count(best_value) == 1: return counts return "" async def _search_method2(self, texts, answers, reverse): """ Return the answer with the maximum/minimum number of keyword occurrences in the texts. :param texts: List of text to analyze :param answers: List of answers :param reverse: True if the best answer occurs the least, False otherwise :return: Answer whose keywords occur most/least in the texts """ print("Running method 2") counts = {answer: {keyword: 0 for keyword in search.find_keywords(answer)} for answer in answers} for text in texts: for keyword_counts in counts.values(): for keyword in keyword_counts: keyword_counts[keyword] += len(re.findall(f" {keyword} ", text)) print(counts) counts_sum = {answer: sum(keyword_counts.values()) for answer, keyword_counts in counts.items()} if not all(c == 0 for c in counts_sum.values()): return counts_sum return ""