Skip to content

Commit

Permalink
(feat) opts
Browse files Browse the repository at this point in the history
  • Loading branch information
@kevincobain2000
kevincobain2000 committed May 6, 2024
1 parent e3e1d06 commit 253516d
Show file tree
Hide file tree
Showing 9 changed files with 104 additions and 76 deletions.
45 changes: 24 additions & 21 deletions frontend/src/components/Extractor.astro
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,35 +55,35 @@ const baseURL = import.meta.env.PUBLIC_BASE_URL;
</div>
</div>

<span x-text="loading.error" class="text-red-500 text-sm text-center block mt-5"></span>
<template x-if="loading.counterCrawls > 0">
<div class="text-center text-sm pt-5">
<span class="">CRAWLS #</span><span x-text="loading.counterCrawls"
class=" text-green-300 font-bold"></span>
<span class="pl-5">HITS #</span><span x-text="loading.counterHits"
class=" text-green-500 font-bold"></span>
</div>
<div class="text-center text-sm pt-1 block">
➠ <span x-text="loading.url" class="text-blue-300"></span>
</div>
</template>
<span x-text="loading.error" class="text-red-300 text-sm text-center block mt-5"></span>
<div class="text-center text-sm pt-5">
<span class="">CRAWLS #</span><span x-text="loading.counterCrawls"
class=" text-green-300 font-bold"></span>
<span class="pl-5">HITS #</span><span x-text="loading.counterHits"
class=" text-green-500 font-bold"></span>
</div>
<div class="text-center text-sm pt-1 block">
➠ <span x-text="loading.url" class="text-blue-300"></span>
</div>
<template x-if="result.data.length > 0">
<div class="overflow-x-auto mt-10 rounded xl:mr-32 xl:ml-32">
<table class="w-full text-sm font-mono text-left text-gray-400">
<thead class="text-xs uppercase bg-gray-700 text-gray-400">
<tr>
<th scope="col" class="px-6 py-3">#</th>
<th scope="col" class="px-6 py-3">Email</th>
<th scope="col" class="px-6 py-3">URL</th>
<th scope="col" class="px-3 py-3">Item No.</th>
<th scope="col" class="px-3 py-3">Email</th>
<th scope="col" class="px-3 py-3">URL</th>
</tr>
</thead>
<tbody>
<template x-for="(item, index) in result.data">
<tr class="border-b bg-gray-800 border-gray-700">
<td class="px-6 py-4" x-text="result.data.length - index"> </td>
<th scope="row" class="px-6 py-4 font-medium text-white" x-text="item.email">
<td class="px-3 py-2" x-text="result.data.length - index"> </td>
<th scope="row" class="px-3 py-2 font-medium text-white" x-text="item.email">
</th>
<td class="px-6 py-4" x-text="item.url"> </td>
<td class="px-3 py-2">
<a x-bind:href="item.url" target="_blank" class="text-blue-300" x-text="item.uri"></a>
</td>
</tr>
</template>
</tbody>
Expand Down Expand Up @@ -148,7 +148,8 @@ const baseURL = import.meta.env.PUBLIC_BASE_URL;
if (!response.ok) {
loading.fetching = false;
loading.url = "";
loading.error = "Error: (" + response.status + ") " + response.statusText;
json = await response.json();
loading.error = "Error: (" + response.status + ") " + response.statusText + " (" + json.error + ")";
return;
}
const reader = response.body.getReader();
Expand All @@ -171,9 +172,10 @@ const baseURL = import.meta.env.PUBLIC_BASE_URL;
if (split.length !== 2) {
continue;
}

if (split[0] == "status") {
loading.counterCrawls++;
loading.url = split[1];
loading.url = split[1].replace(/.*\/\/[^\/]*/, '');
}
if (!split[0].includes("@")) {
continue;
Expand All @@ -188,7 +190,8 @@ const baseURL = import.meta.env.PUBLIC_BASE_URL;
loading.counterHits++;
result.data = [{
email: split[0],
url: split[1]
url: split[1],
uri: split[1].replace(/.*\/\/[^\/]*/, ''),
}, ...result.data];
}
}
Expand Down
14 changes: 11 additions & 3 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,20 @@ func main() {
}

if f.url == "https://" {
pkg.StartEcho(pkg.NewEcho(f.baseURL, publicDir, f.cors), f.host, f.port)
options := []pkg.EchoOption{
func(opt *pkg.EchoOptions) error {
opt.BaseURL = f.baseURL
opt.PublicDir = publicDir
opt.Cors = f.cors
return nil
},
}
pkg.StartEcho(pkg.NewEcho(options), f.host, f.port)
return
}

options := []pkg.Option{
func(opt *pkg.Options) error {
options := []pkg.CrawlOption{
func(opt *pkg.CrawlOptions) error {
opt.TimeoutMillisecond = f.timeout
opt.SleepMillisecond = f.sleep
opt.LimitUrls = f.limitUrls
Expand Down
13 changes: 13 additions & 0 deletions pkg/assets_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ type AssetsHandler struct {
publicDir embed.FS
}

const (
distDir = "frontend/dist"
)

func NewAssetsHandler(publicDir embed.FS, filename string) *AssetsHandler {
return &AssetsHandler{
publicDir: publicDir,
Expand All @@ -38,3 +42,12 @@ func (h *AssetsHandler) GetHTML(c echo.Context) error {
}
return ResponseHTML(c, content, "0")
}

func (h *AssetsHandler) GetIco(c echo.Context) error {
filename := fmt.Sprintf("%s/%s", distDir, h.filename)
content, err := h.publicDir.ReadFile(filename)
if err != nil {
return c.String(http.StatusOK, os.Getenv("VERSION"))
}
return ResponseIco(c, content, "0")
}
10 changes: 5 additions & 5 deletions pkg/crawl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
"github.com/labstack/echo/v4"
)

type Options struct {
type CrawlOptions struct {
TimeoutMillisecond int64
SleepMillisecond int64
URL string
Expand All @@ -26,7 +26,7 @@ type Options struct {
WriteToFile string
}

type Option func(*Options) error
type CrawlOption func(*CrawlOptions) error

type HTTPChallenge struct {
browse *browser.Browser
Expand All @@ -35,11 +35,11 @@ type HTTPChallenge struct {
Emails []string
TotalURLsCrawled int
TotalURLsFound int
options *Options
options *CrawlOptions
}

func NewHTTPChallenge(opts ...Option) *HTTPChallenge {
opt := &Options{}
func NewHTTPChallenge(opts ...CrawlOption) *HTTPChallenge {
opt := &CrawlOptions{}
for _, o := range opts {
err := o(opt)
if err != nil {
Expand Down
23 changes: 19 additions & 4 deletions pkg/echo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,22 @@ import (
"github.com/labstack/echo/v4/middleware"
)

func NewEcho(baseURL string, publicDir embed.FS, cors string) *echo.Echo {
type EchoOptions struct {
BaseURL string
PublicDir embed.FS
Cors string
}

type EchoOption func(*EchoOptions) error

func NewEcho(options []EchoOption) *echo.Echo {
opt := &EchoOptions{}
for _, o := range options {
err := o(opt)
if err != nil {
panic(err)
}
}
e := echo.New()

e.HTTPErrorHandler = HTTPErrorHandler
Expand All @@ -19,10 +34,10 @@ func NewEcho(baseURL string, publicDir embed.FS, cors string) *echo.Echo {
e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{
Format: ltsv(),
}))
SetupRoutes(e, baseURL, publicDir)
SetupRoutes(e, options)

if cors != "" {
SetupCors(e, cors)
if opt.Cors != "" {
SetupCors(e, opt.Cors)
}

return e
Expand Down
15 changes: 13 additions & 2 deletions pkg/extract_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,15 @@ import (

type ExtractHandler struct {
Extractor *Extract
queue int
}

const limitQueue = 20

func NewExtractHandler() *ExtractHandler {
return &ExtractHandler{
Extractor: NewExtract(),
queue: 1,
}
}

Expand All @@ -24,6 +28,11 @@ type ExtractorRequest struct {
}

func (h *ExtractHandler) Get(c echo.Context) error {
h.queue++
if h.queue > limitQueue {
return echo.NewHTTPError(http.StatusUnprocessableEntity, "queue is full, please try again later")
}

req := new(ExtractorRequest)
if err := BindRequest(c, req); err != nil {
return echo.NewHTTPError(http.StatusUnprocessableEntity, err)
Expand All @@ -39,8 +48,8 @@ func (h *ExtractHandler) Get(c echo.Context) error {
c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
c.Response().WriteHeader(http.StatusOK)

opts := []Option{
func(o *Options) error {
opts := []CrawlOption{
func(o *CrawlOptions) error {
o.TimeoutMillisecond = 1000
o.SleepMillisecond = 0
o.URL = req.URL
Expand All @@ -58,6 +67,8 @@ func (h *ExtractHandler) Get(c echo.Context) error {

hc.CrawlRecursiveStream(req.URL, c, enc)

h.queue--

return nil
}

Expand Down
30 changes: 0 additions & 30 deletions pkg/favicon_handler.go
View file

This file was deleted.

4 changes: 4 additions & 0 deletions pkg/request.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,10 @@ func ResponseHTML(c echo.Context, b []byte, cacheMS string) error {
SetHeadersResponseHTML(c.Response().Header(), cacheMS)
return c.Blob(http.StatusOK, "text/html", b)
}
func ResponseIco(c echo.Context, b []byte, cacheMS string) error {
SetHeadersResponseHTML(c.Response().Header(), cacheMS)
return c.Blob(http.StatusOK, "image/x-icon", b)
}
func ResponsePlain(c echo.Context, b []byte, cacheMS string) error {
SetHeadersResponsePlainText(c.Response().Header(), cacheMS)
return c.Blob(http.StatusOK, "text/plain", b)
Expand Down
26 changes: 15 additions & 11 deletions pkg/routes.go
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,23 @@
package pkg

import (
"embed"

"github.com/labstack/echo/v4"
"github.com/labstack/echo/v4/middleware"
)

const (
distDir = "frontend/dist"
faviconFile = "favicon.ico"
)

func SetupRoutes(e *echo.Echo, baseURL string, publicDir embed.FS) {
e.GET(baseURL+"", NewAssetsHandler(publicDir, "index.html").GetHTML)
e.GET(baseURL+"extract", NewAssetsHandler(publicDir, "extract/index.html").GetHTML)
e.GET(baseURL+"api/extract", NewExtractHandler().Get, middleware.RateLimiter(middleware.NewRateLimiterMemoryStore(50)))
func SetupRoutes(e *echo.Echo, options []EchoOption) {
opt := &EchoOptions{}
for _, o := range options {
err := o(opt)
if err != nil {
panic(err)
}
}
e.GET(opt.BaseURL+"", NewAssetsHandler(opt.PublicDir, "index.html").GetHTML)
e.GET(opt.BaseURL+"extract", NewAssetsHandler(opt.PublicDir, "extract/index.html").GetHTML)
e.GET(opt.BaseURL+"favicon.ico", NewAssetsHandler(opt.PublicDir, "favicon.ico").GetIco)
e.GET(opt.BaseURL+"api/extract",
NewExtractHandler().Get,
middleware.RateLimiter(middleware.NewRateLimiterMemoryStore(50)),
)
}

0 comments on commit 253516d

Please sign in to comment.