Source code for

# SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
"""This is the implementation of the Google WEB engine.  Some of this
implementations (manly the :py:obj:`get_google_info`) are shared by other

- :ref:`google images engine`
- :ref:`google news engine`
- :ref:`google videos engine`
- :ref:`google scholar engine`
- :ref:`google autocomplete`


from typing import TYPE_CHECKING

import re
from urllib.parse import urlencode
from lxml import html
import babel
import babel.core
import babel.languages

from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
from searx.locales import language_tag, region_tag, get_offical_locales
from searx import network
from searx.exceptions import SearxEngineCaptchaException
from searx.enginelib.traits import EngineTraits

    import logging

    logger: logging.Logger

traits: EngineTraits

# about
about = {
    "website": '',
    "wikidata_id": 'Q9366',
    "official_api_documentation": '',
    "use_official_api": False,
    "require_api_key": False,
    "results": 'HTML',

# engine dependent config
categories = ['general', 'web']
paging = True
time_range_support = True
safesearch = True

time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}

# Filter results. 0: None, 1: Moderate, 2: Strict
filter_mapping = {0: 'off', 1: 'medium', 2: 'high'}

# specific xpath variables
# ------------------------

results_xpath = './/div[contains(@jscontroller, "SC7lYd")]'
title_xpath = './/a/h3[1]'
href_xpath = './/a[h3]/@href'
content_xpath = './/div[@data-sncf]'

# Suggestions are links placed in a *card-section*, we extract only the text
# from the links not the links itself.
suggestion_xpath = '//div[contains(@class, "EIaa9b")]//a'

# UI_ASYNC = 'use_ac:true,_fmt:html' # returns a HTTP 500 when user search for
#                                    # celebrities like '!google natasha allegri'
#                                    # or '!google chris evans'
UI_ASYNC = 'use_ac:true,_fmt:prog'
"""Format of the response from UI's async request."""

[docs]def get_google_info(params, eng_traits): """Composing various (language) properties for the google engines (:ref:`google API`). This function is called by the various google engines (:ref:`google web engine`, :ref:`google images engine`, :ref:`google news engine` and :ref:`google videos engine`). :param dict param: Request parameters of the engine. At least a ``searxng_locale`` key should be in the dictionary. :param eng_traits: Engine's traits fetched from google preferences (:py:obj:`searx.enginelib.traits.EngineTraits`) :rtype: dict :returns: Py-Dictionary with the key/value pairs: language: The language code that is used by google (e.g. ``lang_en`` or ``lang_zh-TW``) country: The country code that is used by google (e.g. ``US`` or ``TW``) locale: A instance of :py:obj:`babel.core.Locale` build from the ``searxng_locale`` value. subdomain: Google subdomain :py:obj:`google_domains` that fits to the country code. params: Py-Dictionary with additional request arguments (can be passed to :py:func:`urllib.parse.urlencode`). - ``hl`` parameter: specifies the interface language of user interface. - ``lr`` parameter: restricts search results to documents written in a particular language. - ``cr`` parameter: restricts search results to documents originating in a particular country. - ``ie`` parameter: sets the character encoding scheme that should be used to interpret the query string ('utf8'). - ``oe`` parameter: sets the character encoding scheme that should be used to decode the XML result ('utf8'). headers: Py-Dictionary with additional HTTP headers (can be passed to request's headers) - ``Accept: '*/*`` """ ret_val = { 'language': None, 'country': None, 'subdomain': None, 'params': {}, 'headers': {}, 'cookies': {}, 'locale': None, } sxng_locale = params.get('searxng_locale', 'all') try: locale = babel.Locale.parse(sxng_locale, sep='-') except babel.core.UnknownLocaleError: locale = None eng_lang = eng_traits.get_language(sxng_locale, 'lang_en') lang_code = eng_lang.split('_')[-1] # lang_zh-TW --> zh-TW / lang_en --> en country = eng_traits.get_region(sxng_locale, eng_traits.all_locale) # Test zh_hans & zh_hant --> in the topmost links in the result list of list # TW and HK you should a find zh_hant link. In the result # list of zh-CN should not be no hant link instead you should find # somewhere in the top. # '!go 日 :zh-TW' --> # '!go 日 :zh-CN' --> ret_val['language'] = eng_lang ret_val['country'] = country ret_val['locale'] = locale ret_val['subdomain'] = eng_traits.custom['supported_domains'].get(country.upper(), '') # hl parameter: # The hl parameter specifies the interface language (host language) of # your user interface. To improve the performance and the quality of your # search results, you are strongly encouraged to set this parameter # explicitly. # # The Interface Language: # ret_val['params']['hl'] = lang_code # lr parameter: # The lr (language restrict) parameter restricts search results to # documents written in a particular language. # # Language Collection Values: # # # To select 'all' languages an empty 'lr' value is used. # # Different to other google services, Google Schloar supports to select more # than one language. The languages are seperated by a pipe '|' (logical OR). # By example: &lr=lang_zh-TW%7Clang_de selects articles written in # traditional chinese OR german language. ret_val['params']['lr'] = eng_lang if sxng_locale == 'all': ret_val['params']['lr'] = '' # cr parameter: # The cr parameter restricts search results to documents originating in a # particular country. # ret_val['params']['cr'] = 'country' + country if sxng_locale == 'all': ret_val['params']['cr'] = '' # gl parameter: (mandatory by Geeogle News) # The gl parameter value is a two-letter country code. For WebSearch # results, the gl parameter boosts search results whose country of origin # matches the parameter value. See the Country Codes section for a list of # valid values. # Specifying a gl parameter value in WebSearch requests should improve the # relevance of results. This is particularly true for international # customers and, even more specifically, for customers in English-speaking # countries other than the United States. # ret_val['params']['gl'] = country # ie parameter: # The ie parameter sets the character encoding scheme that should be used # to interpret the query string. The default ie value is latin1. # ret_val['params']['ie'] = 'utf8' # oe parameter: # The oe parameter sets the character encoding scheme that should be used # to decode the XML result. The default oe value is latin1. # ret_val['params']['oe'] = 'utf8' # num parameter: # The num parameter identifies the number of search results to return. # The default num value is 10, and the maximum value is 20. If you request # more than 20 results, only 20 results will be returned. # # HINT: seems to have no effect (tested in google WEB & Images) # ret_val['params']['num'] = 20 # HTTP headers ret_val['headers']['Accept'] = '*/*' # Cookies # - # - ret_val['cookies']['CONSENT'] = "YES+" return ret_val
def detect_google_sorry(resp): if == '' or resp.url.path.startswith('/sorry'): raise SearxEngineCaptchaException()
[docs]def request(query, params): """Google search request""" # pylint: disable=line-too-long offset = (params['pageno'] - 1) * 10 google_info = get_google_info(params, traits) # query_url = ( 'https://' + google_info['subdomain'] + '/search' + "?" + urlencode( { 'q': query, **google_info['params'], 'filter': '0', 'start': offset, # 'vet': '12ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0QxK8CegQIARAC..i', # 'ved': '2ahUKEwik3ZbIzfn7AhXMX_EDHbUDBh0Q_skCegQIARAG', # 'cs' : 1, # 'sa': 'N', # 'yv': 3, # 'prmd': 'vin', # 'ei': 'GASaY6TxOcy_xc8PtYeY6AE', # 'sa': 'N', # 'sstk': 'AcOHfVkD7sWCSAheZi-0tx_09XDO55gTWY0JNq3_V26cNN-c8lfD45aZYPI8s_Bqp8s57AHz5pxchDtAGCA_cikAWSjy9kw3kgg' # formally known as use_mobile_ui 'asearch': 'arc', 'async': UI_ASYNC, } ) ) if params['time_range'] in time_range_dict: query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]}) if params['safesearch']: query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]}) params['url'] = query_url params['cookies'] = google_info['cookies'] params['headers'].update(google_info['headers']) return params
# =26;[3,"dimg_ZNMiZPCqE4apxc8P3a2tuAQ_137"]a87;data:image/jpeg;base64,/9j/4AAQSkZJRgABA # ...6T+9Nl4cnD+gr9OK8I56/tX3l86nWYw//2Q==26; RE_DATA_IMAGE = re.compile(r'"(dimg_[^"]*)"[^;]*;(data:image[^;]*;[^;]*);') def _parse_data_images(dom): data_image_map = {} for img_id, data_image in RE_DATA_IMAGE.findall(dom.text_content()): end_pos = data_image.rfind('=') if end_pos > 0: data_image = data_image[: end_pos + 1] data_image_map[img_id] = data_image logger.debug('data:image objects --> %s', list(data_image_map.keys())) return data_image_map
[docs]def response(resp): """Get response from google's search request""" # pylint: disable=too-many-branches, too-many-statements detect_google_sorry(resp) results = [] # convert the text to dom dom = html.fromstring(resp.text) data_image_map = _parse_data_images(dom) # results --> answer answer_list = eval_xpath(dom, '//div[contains(@class, "LGOjhe")]') if answer_list: answer_list = [_.xpath("normalize-space()") for _ in answer_list] results.append({'answer': ' '.join(answer_list)}) else: logger.debug("did not find 'answer'") # parse results for result in eval_xpath_list(dom, results_xpath): # pylint: disable=too-many-nested-blocks try: title_tag = eval_xpath_getindex(result, title_xpath, 0, default=None) if title_tag is None: # this not one of the common google results *section* logger.debug('ignoring item from the result_xpath list: missing title') continue title = extract_text(title_tag) url = eval_xpath_getindex(result, href_xpath, 0, None) if url is None: logger.debug('ignoring item from the result_xpath list: missing url of title "%s"', title) continue content_nodes = eval_xpath(result, content_xpath) content = extract_text(content_nodes) if not content: logger.debug('ignoring item from the result_xpath list: missing content of title "%s"', title) continue img_src = content_nodes[0].xpath('.//img/@src') if img_src: img_src = img_src[0] if img_src.startswith('data:image'): img_id = content_nodes[0].xpath('.//img/@id') if img_id: img_src = data_image_map.get(img_id[0]) else: img_src = None results.append({'url': url, 'title': title, 'content': content, 'img_src': img_src}) except Exception as e: # pylint: disable=broad-except logger.error(e, exc_info=True) continue # parse suggestion for suggestion in eval_xpath_list(dom, suggestion_xpath): # append suggestion results.append({'suggestion': extract_text(suggestion)}) # return results return results
# get supported languages from their site skip_countries = [ # official language of google-country not in google-languages 'AL', # Albanien (sq) 'AZ', # Aserbaidschan (az) 'BD', # Bangladesch (bn) 'BN', # Brunei Darussalam (ms) 'BT', # Bhutan (dz) 'ET', # Äthiopien (am) 'GE', # Georgien (ka, os) 'GL', # Grönland (kl) 'KH', # Kambodscha (km) 'LA', # Laos (lo) 'LK', # Sri Lanka (si, ta) 'ME', # Montenegro (sr) 'MK', # Nordmazedonien (mk, sq) 'MM', # Myanmar (my) 'MN', # Mongolei (mn) 'MV', # Malediven (dv) // dv_MV is unknown by babel 'MY', # Malaysia (ms) 'NP', # Nepal (ne) 'TJ', # Tadschikistan (tg) 'TM', # Turkmenistan (tk) 'UZ', # Usbekistan (uz) ]
[docs]def fetch_traits(engine_traits: EngineTraits, add_domains: bool = True): """Fetch languages from Google.""" # pylint: disable=import-outside-toplevel, too-many-branches engine_traits.custom['supported_domains'] = {} resp = network.get('') if not resp.ok: raise RuntimeError("Response from Google's preferences is not OK.") dom = html.fromstring(resp.text) # supported language codes lang_map = {'no': 'nb'} for x in eval_xpath_list(dom, '//*[@id="langSec"]//input[@name="lr"]'): eng_lang = x.get("value").split('_')[-1] try: locale = babel.Locale.parse(lang_map.get(eng_lang, eng_lang), sep='-') except babel.UnknownLocaleError: print("ERROR: %s -> %s is unknown by babel" % (x.get("data-name"), eng_lang)) continue sxng_lang = language_tag(locale) conflict = engine_traits.languages.get(sxng_lang) if conflict: if conflict != eng_lang: print("CONFLICT: babel %s --> %s, %s" % (sxng_lang, conflict, eng_lang)) continue engine_traits.languages[sxng_lang] = 'lang_' + eng_lang # alias languages engine_traits.languages['zh'] = 'lang_zh-CN' # supported region codes for x in eval_xpath_list(dom, '//*[@name="region"]/..//input[@name="region"]'): eng_country = x.get("value") if eng_country in skip_countries: continue if eng_country == 'ZZ': engine_traits.all_locale = 'ZZ' continue sxng_locales = get_offical_locales(eng_country, engine_traits.languages.keys(), regional=True) if not sxng_locales: print("ERROR: can't map from google country %s (%s) to a babel region." % (x.get('data-name'), eng_country)) continue for sxng_locale in sxng_locales: engine_traits.regions[region_tag(sxng_locale)] = eng_country # alias regions engine_traits.regions['zh-CN'] = 'HK' # supported domains if add_domains: resp = network.get('') if not resp.ok: raise RuntimeError("Response from is not OK.") for domain in resp.text.split(): domain = domain.strip() if not domain or domain in [ '', ]: continue region = domain.split('.')[-1].upper() engine_traits.custom['supported_domains'][region] = 'www' + domain if region == 'HK': # There is no, we use for zh-CN engine_traits.custom['supported_domains']['CN'] = 'www' + domain