Use consistent header for all result types (#535)

Introduces a header for switching between result types (i.e. "All", "News", etc) that is consistent between the different result types. Previously, image results had a tab header that was formatted in a drastically different manner, which was jarring when switching from a different result page to the Images page. Created a G class enum to reference class names returned in search results. As noted in the class doc, this should only be used/updated as a last resort, as class names change frequently. For some instances, such as replacing the tbm tab, it's a lot easier to just replace by header name than attempting to replace it based on how the element is structured. Also updated a few styles to revert the latest styling changes being applied by Google. Co-authored-by: jacr13 <ramos.joao@protonmail.com> Co-authored-by: Ben Busby <contact@benbusby.com>
benbusby · Feb 7, 2022 · 11099f7 · 11099f7
1 parent 4aa94a5
commit 11099f7
Show file tree

Hide file tree

Showing 14 changed files with 821 additions and 138 deletions.
diff --git a/app/__init__.py b/app/__init__.py
@@ -52,6 +52,9 @@
 app.config['THEMES'] = json.load(open(
     os.path.join(app.config['STATIC_FOLDER'], 'settings/themes.json'),
     encoding='utf-8'))
+app.config['HEADER_TABS'] = json.load(open(
+    os.path.join(app.config['STATIC_FOLDER'], 'settings/header_tabs.json'),
+    encoding='utf-8'))
 app.config['CONFIG_PATH'] = os.getenv(
     'CONFIG_VOLUME',
     os.path.join(app.config['STATIC_FOLDER'], 'config'))

diff --git a/app/filter.py b/app/filter.py
@@ -1,5 +1,6 @@
 from app.models.config import Config
 from app.models.endpoint import Endpoint
+from app.models.g_classes import GClasses
 from app.request import VALID_PARAMS, MAPS_URL
 from app.utils.misc import read_config_bool
 from app.utils.results import *
@@ -13,6 +14,15 @@
 import os
 
 minimal_mode_sections = ['Top stories', 'Images', 'People also ask']
+unsupported_g_pages = [
+    'support.google.com',
+    'accounts.google.com',
+    'policies.google.com',
+    'google.com/preferences',
+    'google.com/intl',
+    'advanced_search',
+    'tbm=shop'
+]
 
 
 def extract_q(q_str: str, href: str) -> str:
@@ -80,6 +90,7 @@ def clean(self, soup) -> BeautifulSoup:
         self.remove_block_url()
         self.collapse_sections()
         self.update_styling(soup)
+        self.remove_block_tabs(soup)
 
         for img in [_ for _ in soup.find_all('img') if 'src' in _.attrs]:
             self.update_element_src(img, 'image/png')
@@ -143,6 +154,21 @@ def remove_block_url(self) -> None:
                           if block_url.search(_.attrs['href']) is not None]
             _ = div.decompose() if len(block_divs) else None
 
+    def remove_block_tabs(self, soup) -> None:
+        if self.main_divs:
+            for div in self.main_divs.find_all(
+                'div',
+                attrs={'class': f'{GClasses.main_tbm_tab}'}
+            ):
+                _ = div.decompose()
+        else:
+            # when in images tab
+            for div in soup.find_all(
+                'div',
+                attrs={'class': f'{GClasses.images_tbm_tab}'}
+            ):
+                _ = div.decompose()
+
     def collapse_sections(self) -> None:
         """Collapses long result sections ("people also asked", "related
          searches", etc) into "details" elements
@@ -273,6 +299,26 @@ def update_styling(self, soup) -> None:
         except AttributeError:
             pass
 
+        # Fix body max width on images tab
+        style = soup.find('style')
+        div = soup.find('div', attrs={'class': f'{GClasses.images_tbm_tab}'})
+        if style and div and not self.mobile:
+            css = style.string
+            css_html_tag = (
+                'html{'
+                'font-family: Roboto, Helvetica Neue, Arial, sans-serif;'
+                'font-size: 14px;'
+                'line-height: 20px;'
+                'text-size-adjust: 100%;'
+                'word-wrap: break-word;'
+                '}'
+            )
+            css = f"{css_html_tag}{css}"
+            css = re.sub('body{(.*?)}',
+                         'body{padding:0 8px;margin:0 auto;max-width:736px;}',
+                         css)
+            style.string = css
+
     def update_link(self, link: Tag) -> None:
         """Update internal link paths with encrypted path, otherwise remove
         unnecessary redirects and/or marketing params from the url
@@ -284,14 +330,15 @@ def update_link(self, link: Tag) -> None:
             None (the tag is updated directly)
 
         """
-        # Replace href with only the intended destination (no "utm" type tags)
-        href = link['href'].replace('https://www.google.com', '')
-        if 'advanced_search' in href or 'tbm=shop' in href:
+        # Remove any elements that direct to unsupported Google pages
+        if any(url in link['href'] for url in unsupported_g_pages):
             # FIXME: The "Shopping" tab requires further filtering (see #136)
             # Temporarily removing all links to that tab for now.
             link.decompose()
             return
 
+        # Replace href with only the intended destination (no "utm" type tags)
+        href = link['href'].replace('https://www.google.com', '')
         result_link = urlparse.urlparse(href)
         q = extract_q(result_link.query, href)
 
@@ -362,11 +409,8 @@ def view_image(self, soup) -> BeautifulSoup:
         """
 
         # get some tags that are unchanged between mobile and pc versions
-        search_input = soup.find_all('td', attrs={'class': "O4cRJf"})[0]
-        search_options = soup.find_all('div', attrs={'class': "M7pB2"})[0]
         cor_suggested = soup.find_all('table', attrs={'class': "By0U9"})
         next_pages = soup.find_all('table', attrs={'class': "uZgmoc"})[0]
-        information = soup.find_all('div', attrs={'class': "TuS8Ad"})[0]
 
         results = []
         # find results div
@@ -404,12 +448,7 @@ def view_image(self, soup) -> BeautifulSoup:
                                              results=results,
                                              view_label="View Image"),
                              features='html.parser')
-        # replace search input object
-        soup.find_all('td',
-                      attrs={'class': "O4cRJf"})[0].replaceWith(search_input)
-        # replace search options object (All, Images, Videos, etc.)
-        soup.find_all('div',
-                      attrs={'class': "M7pB2"})[0].replaceWith(search_options)
+
         # replace correction suggested by google object if exists
         if len(cor_suggested):
             soup.find_all(
@@ -419,7 +458,4 @@ def view_image(self, soup) -> BeautifulSoup:
         # replace next page object at the bottom of the page
         soup.find_all('table',
                       attrs={'class': "uZgmoc"})[0].replaceWith(next_pages)
-        # replace information about user connection at the bottom of the page
-        soup.find_all('div',
-                      attrs={'class': "TuS8Ad"})[0].replaceWith(information)
         return soup
diff --git a/app/models/g_classes.py b/app/models/g_classes.py
@@ -0,0 +1,16 @@
+from enum import Enum
+
+
+class GClasses(Enum):
+    """A class for tracking obfuscated class names used in Google results that
+    are directly referenced in Whoogle's filtering code.
+
+    Note: Using these should be a last resort. It is always preferred to filter
+    results using structural cues instead of referencing class names, as these
+    are liable to change at any moment.
+    """
+    main_tbm_tab = 'KP7LCb'
+    images_tbm_tab = 'n692Zd'
+
+    def __str__(self):
+        return self.value
diff --git a/app/routes.py b/app/routes.py
@@ -1,6 +1,7 @@
 import argparse
 import base64
 import io
+import os
 import json
 import os
 import pickle
@@ -17,14 +18,15 @@
 from app.utils.bangs import resolve_bang
 from app.utils.misc import read_config_bool, get_client_ip, get_request_url
 from app.utils.results import add_ip_card, bold_search_terms,\
-    add_currency_card, check_currency
-from app.utils.search import *
+    add_currency_card, check_currency, get_tabs_content
+from app.utils.search import Search, needs_https, has_captcha
 from app.utils.session import generate_user_key, valid_user_session
 from bs4 import BeautifulSoup as bsoup
 from flask import jsonify, make_response, request, redirect, render_template, \
-    send_file, session, url_for
+    send_file, session, url_for, g
 from requests import exceptions, get
 from requests.models import PreparedRequest
+from cryptography.fernet import Fernet
 
 # Load DDG bang json files only on init
 bang_json = json.load(open(app.config['BANG_FILE'])) or {}
@@ -347,6 +349,12 @@ def search():
         html_soup = bsoup(str(response), 'html.parser')
         response = add_ip_card(html_soup, get_client_ip(request))
 
+    # Update tabs content
+    tabs = get_tabs_content(app.config['HEADER_TABS'],
+                            search_util.full_query,
+                            search_util.search_type,
+                            translation)
+
     # Feature to display currency_card
     conversion = check_currency(str(response))
     if conversion:
@@ -373,15 +381,14 @@ def search():
         ) and not search_util.search_type,  # Standard search queries only
         response=response,
         version_number=app.config['VERSION_NUMBER'],
-        search_header=(render_template(
+        search_header=render_template(
             'header.html',
             config=g.user_config,
             logo=render_template('logo.html', dark=g.user_config.dark),
             query=urlparse.unquote(query),
             search_type=search_util.search_type,
-            mobile=g.user_request.mobile)
-                       if 'isch' not in
-                          search_util.search_type else '')), 200
+            mobile=g.user_request.mobile,
+            tabs=tabs))
 
 
 @app.route(f'/{Endpoint.config}', methods=['GET', 'POST', 'PUT'])

diff --git a/app/static/css/dark-theme.css b/app/static/css/dark-theme.css
@@ -74,6 +74,7 @@ select {
 
 .ZINbbc.luh4tb {
     background: var(--whoogle-dark-result-bg) !important;
+    margin-bottom: 24px !important;
 }
 
 .bRsWnc {
@@ -204,3 +205,7 @@ path {
 .cb:focus {
   color: var(--whoogle-dark-contrast-text) !important;
 }
+
+.desktop-header, .mobile-header {
+    background-color: var(--whoogle-dark-result-bg) !important;
+}
diff --git a/app/static/css/header.css b/app/static/css/header.css
@@ -98,6 +98,11 @@ header {
     border: 0px !important;
 }
 
+.autocomplete-mobile{
+    display: -webkit-box;
+    width: 100%;
+}
+
 .desktop-header-logo {
     height: 1.65em;
 }
@@ -106,3 +111,113 @@ header {
     width: 100%;
     flex: 1
 }
+
+a {
+    color: #1967D2;
+    text-decoration: none;
+    tap-highlight-color: rgba(0, 0, 0, .10);
+}
+
+.header-tab-div {
+    border-radius: 0 0 8px 8px;
+    box-shadow: 0 2px 3px rgba(32, 33, 36, 0.18);
+    margin-bottom: 20px;
+    overflow: hidden;
+}
+
+.header-tab-div-2 {
+    border-top: 1px solid #dadce0;
+    height: 39px;
+    overflow: hidden;
+}
+
+.header-tab-div-3 {
+    height: 51px;
+    overflow-x: auto;
+    overflow-y: hidden;
+}
+
+.desktop-header {
+    height: 39px;
+    display: box;
+    display: flex;
+    width: 100%;
+}
+
+.header-tab {
+    box-pack: justify;
+    font-size: 14px;
+    line-height: 37px;
+    justify-content: space-between;
+}
+
+.desktop-header a, .desktop-header span {
+    color: #70757a;
+    display: block;
+    flex: none;
+    padding: 0 16px;
+    text-align: center;
+    text-transform: uppercase;
+}
+
+span.header-tab-span {
+    border-bottom: 2px solid #4285f4;
+    color: #4285f4;
+    font-weight: bold;
+}
+
+.mobile-header {
+    height: 39px;
+    display: box;
+    display: flex;
+    overflow-x: scroll;
+    width: 100%;
+    padding-left: 12px;
+}
+
+.mobile-header a, .mobile-header span {
+    color: #70757a;
+    text-decoration: none;
+    display: inline-block;
+    /* padding: 8px 12px 8px 12px; */
+}
+
+span.mobile-tab-span {
+    border-bottom: 2px solid #202124;
+    color: #202124;
+    height: 26px;
+    /* margin: 0 12px; */
+    /* padding: 0; */
+}
+
+.desktop-header input {
+    margin: 2px 4px 2px 8px;
+}
+
+a.header-tab-a:visited {
+    color: #70757a;
+}
+
+.header-tab-div-end {
+    border-left: 1px solid rgba(0, 0, 0, 0.12);
+}
+
+.search-bar-input {
+    display: block;
+    font-size: 16px;
+    padding: 0 0 0 8px;
+    flex: 1;
+    height: 35px;
+    outline: none;
+    border: none;
+    width: 100%;
+    -webkit-tap-highlight-color: rgba(0, 0, 0, 0);
+    overflow: hidden;
+}
+
+@media (max-width: 801px) {
+    .header-tab-div {
+        margin-bottom: 10px !important
+    }
+}
+
diff --git a/app/static/css/light-theme.css b/app/static/css/light-theme.css
@@ -43,6 +43,8 @@ select {
 
 .ZINbbc.luh4tb {
     background: var(--whoogle-result-bg) !important;
+    box-shadow: 0 1px 6px rgba(32,33,36,0.28) !important;
+    margin-bottom: 24px !important;
 }
 
 .bRsWnc {
@@ -190,3 +192,7 @@ path {
 .cb:focus {
   color: var(--whoogle-text) !important;
 }
+
+.desktop-header, .mobile-header {
+    background-color: var(--whoogle-result-bg) !important;
+}