webpage simplifier for neper

app/Controllers/Http/ApiController.ts

@@ -0,0 +1,36 @@
+import type { HttpContextContract } from '@ioc:Adonis/Core/HttpContext'
+import { inject } from '@adonisjs/fold'
+import fetch from 'isomorphic-fetch'
+
+import {WebpageSimplifierService} from "App/Services/Api/WebpageSimplifierService"
+
+
+const DOMAIN = process.env.NODE_ENV === 'development' ? 'localhost:3333' : 'paxo.fr'
+
+@inject()
+export default class ApiController {
+  constructor(private wpSimplifierService: WebpageSimplifierService) {
+  }
+
+  public async index({ response }: HttpContextContract){
+    return response.redirect().toPath(`http://${DOMAIN}/`)
+  }
+
+  public async webpageSimplifier({ request, response, view }: HttpContextContract){
+    const url = request.qs().url
+    const version = request.qs().v ? request.qs().v : '0.1-alpha.0'
+    const isNeperRequest = request.qs().neper === 'true'
+    try {
+      const responseHtml = await (await fetch(url)).text()
+      const html = this.wpSimplifierService.simplify(responseHtml)
+
+      return view.render('api/webpage-simplifier/index', {
+        html: html,
+        version: version,
+        isNeperRequest: isNeperRequest
+      })
+    } catch (err) {
+      return response.badRequest('error: please provide a correct url')
+    }
+  }
+}

app/Services/Api/WebpageSimplifierService.ts

@@ -0,0 +1,85 @@
+import * as himalaya from "himalaya"
+import {Element, TreeElement} from "App/Services/Api/WebpageSimplifierTypes"
+
+export class WebpageSimplifierService {
+  private unusedTagTypes: string[] = [
+    'link',
+    'script',
+    'style'
+  ]
+
+  private unusedAttrs: string[] = [
+    'class'
+  ]
+
+  private parse(html: string): Element[] {
+    function removeNewlines(node) {
+      if (node.type === 'text' && /^\s*$/.test(node.content)) {
+        return null // Remove the node
+      }
+
+      if (node.children) {
+        node.children = node.children.map(removeNewlines).filter(Boolean);
+      }
+
+      return node
+    }
+
+    return himalaya.parse(html).map(removeNewlines).filter(Boolean)
+  }
+
+  private doSimplify(html: Element[], lastElement: Element = {type: 'element', tagName: 'root', attributes: []}): TreeElement | any[] | undefined {
+    let newTree: Array<TreeElement | any[] | undefined> = []
+
+    for(const element of html) {
+      const tagName = element.tagName ? element.tagName : ''
+      if(element.children !== undefined && !(this.unusedTagTypes.includes(tagName))) {
+        newTree.push(this.doSimplify(element.children, element))
+      } else if(element.type === 'text') {
+        if(lastElement.attributes) {
+          for(const attr of lastElement.attributes) {
+            if(this.unusedAttrs.includes(attr.key)) {
+              const index = lastElement.attributes.indexOf(attr);
+              if (index !== -1) {
+                lastElement.attributes.splice(index, 1);
+              }
+            }
+          }
+        }
+        const newTreeElement: TreeElement = {parent: lastElement, content: element.content}
+        newTree.push(newTreeElement)
+      }
+    }
+    if(newTree.length !== 0) return newTree.length === 1 ? newTree[0] : newTree
+  }
+
+  private generateHtml(tree: any[] | undefined): string {
+    if(tree === undefined) return ''
+
+    let html: string = ''
+
+    for(const element of tree) {
+      if(element) {
+        if(Array.isArray(element)) {
+          html += this.generateHtml(element)
+        } else {
+          let attrs = ''
+          if(element.parent.attributes) {
+            for(const attr of element.parent.attributes) {
+              attrs += ` ${attr.key}="${attr.value}"`
+            }
+            html += `<${element.parent.tagName}${attrs}>${element.content}</${element.parent.tagName}>`
+          }
+        }
+      }
+    }
+    return html
+  }
+
+  public simplify(responseHtml: string): string {
+    const beforeHtml = this.parse(responseHtml)
+    const afterHtml = this.doSimplify(beforeHtml)
+
+    return this.generateHtml([afterHtml])
+  }
+}

app/Services/Api/WebpageSimplifierTypes.ts

@@ -0,0 +1,18 @@
+
+export declare interface Attribute {
+  key: string
+  value: string
+}
+
+export declare interface Element {
+  type: string & ('element' | 'text')
+  tagName?: string
+  attributes?: Attribute[]
+  children?: Element[]
+  content?: string
+}
+
+export declare interface TreeElement {
+  parent: Element
+  content: string | undefined
+}

package.json

@@ -52,6 +52,8 @@
     "@adonisjs/session": "^6.2.0",
     "@adonisjs/shield": "^7.0.0",
     "@adonisjs/view": "^6.1.0",
+    "himalaya": "^1.1.0",
+    "isomorphic-fetch": "^3.0.0",
     "luxon": "^3.4.0",
     "mysql2": "^3.6.0",
     "phc-argon2": "^1.1.4",

resources/views/api/webpage-simplifier/index.edge

@@ -0,0 +1,28 @@
+<!DOCTYPE html>
+<head>
+    <meta charset="UTF-8">
+    <meta name="generator" content="Paxo webpage-simplifier v{{ version }}">
+</head>
+<body>
+    @if(!isNeperRequest)
+      <style>
+          #banner {
+            display: flex;
+            justify-content: space-between;
+            align-content: center;
+            padding-inline: 10px;
+            font-family: Monaco, monospace;
+          }
+
+          #banner a {
+            text-decoration: none;
+          }
+      </style>
+
+      <div id="banner">
+          <p>Paxo webpage-simplifier v{{ version }} - <a href="https://github.com/paxo-phone/Neper">Neper</a> browser</p>
+          <a href="https://paxo.fr">Paxo</a>
+      </div>
+    @endif
+    {{{ html }}}
+</body>

start/routes.ts

@@ -21,6 +21,8 @@
 import Route from '@ioc:Adonis/Core/Route'
 
 
+const DOMAIN = process.env.NODE_ENV === 'development' ? 'localhost' : 'paxo.fr'
+
 // ----------------------------------------------
 // Main routes
 // ----------------------------------------------
@@ -31,6 +33,7 @@ Route.group(() => {
   Route.get('projects', 'CoreController.projects')
   Route.get('press', 'PressController.index')
 })
+  .domain(DOMAIN)
 
 // ----------------------------------------------
 // Tutorials
@@ -42,6 +45,7 @@ Route.group(() => {
   Route.get('/t/:id/end', 'TutorialsController.stepEnd').as('tutorials.stepEnd')
 })
   .prefix('/tutorials')
+  .domain(DOMAIN)
 
 // ----------------------------------------------
 // Auth
@@ -69,14 +73,17 @@ Route.group(() => {
 
   Route.post('/logout', 'UsersController.logoutProcess')
     .as('auth.logoutProcess')
-}).prefix('/auth')
+})
+  .prefix('/auth')
+  .domain(DOMAIN)
 
 // ----------------------------------------------
 // Dashboard
 // ----------------------------------------------
 Route.get('/dashboard', 'DashboardController.index')
   .middleware('auth')
   .as('dashboard')
+  .domain(DOMAIN)
 
 // ----------------------------------------------
 // Admin panel
@@ -105,3 +112,16 @@ Route.group(() => {
 })
   .middleware('auth')
   .prefix('/admin-panel')
+  .domain(DOMAIN)
+
+// ----------------------------------------------
+// API
+// ----------------------------------------------
+Route.group(() => {
+  Route.get('/', 'ApiController.index')
+    .as('api.index')
+
+  Route.get('/neper/wp-simplifier', 'ApiController.webpageSimplifier')
+    .as('api.webpageSimplifier.index')
+})
+  .domain(`api.${DOMAIN}`)

yarn.lock

@@ -5169,6 +5169,11 @@ hexoid@^1.0.0:
   resolved "https://registry.yarnpkg.com/hexoid/-/hexoid-1.0.0.tgz#ad10c6573fb907de23d9ec63a711267d9dc9bc18"
   integrity sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==
 
+himalaya@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/himalaya/-/himalaya-1.1.0.tgz#31724ae9d35714cd7c6f4be94888953f3604606a"
+  integrity sha512-LLase1dHCRMel68/HZTFft0N0wti0epHr3nNY7ynpLbyZpmrKMQ8YIpiOV77TM97cNpC8Wb2n6f66IRggwdWPw==
+
 hpack.js@^2.1.6:
   version "2.1.6"
   resolved "https://registry.yarnpkg.com/hpack.js/-/hpack.js-2.1.6.tgz#87774c0949e513f42e84575b3c45681fade2a0b2"
@@ -5653,6 +5658,14 @@ isobject@^3.0.0, isobject@^3.0.1:
   resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df"
   integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==
 
+isomorphic-fetch@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/isomorphic-fetch/-/isomorphic-fetch-3.0.0.tgz#0267b005049046d2421207215d45d6a262b8b8b4"
+  integrity sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==
+  dependencies:
+    node-fetch "^2.6.1"
+    whatwg-fetch "^3.4.1"
+
 jest-diff@^25.5.0:
   version "25.5.0"
   resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-25.5.0.tgz#1dd26ed64f96667c068cef026b677dfa01afcfa9"
@@ -6560,6 +6573,13 @@ node-emoji@^1.11.0:
   dependencies:
     lodash "^4.17.21"
 
+node-fetch@^2.6.1:
+  version "2.7.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
+  integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
+  dependencies:
+    whatwg-url "^5.0.0"
+
 node-fetch@^2.6.7:
   version "2.6.12"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.12.tgz#02eb8e22074018e3d5a83016649d04df0e348fba"
@@ -9064,6 +9084,11 @@ websocket-extensions@>=0.1.1:
   resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.4.tgz#7f8473bc839dfd87608adb95d7eb075211578a42"
   integrity sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==
 
+whatwg-fetch@^3.4.1:
+  version "3.6.19"
+  resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-3.6.19.tgz#caefd92ae630b91c07345537e67f8354db470973"
+  integrity sha512-d67JP4dHSbm2TrpFj8AbO8DnL1JXL5J9u0Kq2xW6d0TFDbCA3Muhdt8orXC22utleTVj7Prqt82baN6RBvnEgw==
+
 whatwg-url@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"