-
Notifications
You must be signed in to change notification settings - Fork 778
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1245 from dogancanbakir/error_page_classifier
Error page classifier
- Loading branch information
Showing
12 changed files
with
392 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| package errorpageclassifier | ||
|
|
||
| import ( | ||
| _ "embed" | ||
|
|
||
| "github.com/jaytaylor/html2text" | ||
| "github.com/projectdiscovery/utils/ml/naive_bayes" | ||
| ) | ||
|
|
||
| //go:embed clf.gob | ||
| var classifierData []byte | ||
|
|
||
| type ErrorPageClassifier struct { | ||
| classifier *naive_bayes.NaiveBayesClassifier | ||
| } | ||
|
|
||
| func New() *ErrorPageClassifier { | ||
| classifier, err := naive_bayes.NewClassifierFromFileData(classifierData) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| return &ErrorPageClassifier{classifier: classifier} | ||
| } | ||
|
|
||
| func (n *ErrorPageClassifier) Classify(html string) string { | ||
| text := htmlToText(html) | ||
| if text == "" { | ||
| return "other" | ||
| } | ||
| return n.classifier.Classify(text) | ||
| } | ||
|
|
||
| func htmlToText(html string) string { | ||
| text, err := html2text.FromString(html, html2text.Options{TextOnly: true}) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| return text | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| package errorpageclassifier | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func TestErrorPageClassifier(t *testing.T) { | ||
| t.Run("test creation of new ErrorPageClassifier", func(t *testing.T) { | ||
| epc := New() | ||
| assert.NotNil(t, epc) | ||
| }) | ||
|
|
||
| t.Run("test classification non error page text", func(t *testing.T) { | ||
| epc := New() | ||
| assert.Equal(t, "nonerror", epc.Classify(`<!DOCTYPE html> | ||
| <html lang="en"> | ||
| <head> | ||
| <meta charset="UTF-8"> | ||
| <title>Terms of Service</title> | ||
| </head> | ||
| <body> | ||
| <h1>Welcome to our Terms of Service page.</h1> | ||
| <p>Understand our conditions for providing services.</p> | ||
| </body> | ||
| </html> | ||
| `)) | ||
| }) | ||
|
|
||
| t.Run("test classification on error page text", func(t *testing.T) { | ||
| epc := New() | ||
| assert.Equal(t, "error", epc.Classify(`<!DOCTYPE html> | ||
| <html> | ||
| <head> | ||
| <title>Error 403: Forbidden</title> | ||
| <style> | ||
| .error-message { | ||
| text-align: center; | ||
| color: #333; | ||
| } | ||
| </style> | ||
| </head> | ||
| <body> | ||
| <div class="error-message"> | ||
| <h1>Error 403: Forbidden</h1> | ||
| <p>Sorry you don't have access rights to this page.</p> | ||
| </div> | ||
| </body> | ||
| </html> | ||
| `)) | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| index=21 | ||
| resume_from=www.hackerone.com |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters