Skip to content

Commit

Permalink
Merge pull request #92 from projectdiscovery/wappalyzer-more-enhancem…
Browse files Browse the repository at this point in the history 
…ents

Wappalyzer more enhancements to public API
  • Loading branch information
@Ice3man543
Ice3man543 committed Jun 12, 2024
2 parents 6a4b83d + 71c54d7 commit c875ee8
Show file tree
Hide file tree
Showing 6 changed files with 12,671 additions and 7,031 deletions.
59 changes: 42 additions & 17 deletions cmd/update-fingerprints/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type Fingerprint struct {
Cats []int `json:"cats"`
CSS interface{} `json:"css"`
Cookies map[string]string `json:"cookies"`
Dom interface{} `json:"dom"`
JS map[string]string `json:"js"`
Headers map[string]string `json:"headers"`
HTML interface{} `json:"html"`
Expand All @@ -49,20 +50,21 @@ type OutputFingerprints struct {

// OutputFingerprint is a single piece of information about a tech validated and normalized
type OutputFingerprint struct {
Cats []int `json:"cats,omitempty"`
CSS []string `json:"css,omitempty"`
Cookies map[string]string `json:"cookies,omitempty"`
JS []string `json:"js,omitempty"`
Headers map[string]string `json:"headers,omitempty"`
HTML []string `json:"html,omitempty"`
Script []string `json:"scripts,omitempty"`
ScriptSrc []string `json:"scriptSrc,omitempty"`
Meta map[string][]string `json:"meta,omitempty"`
Implies []string `json:"implies,omitempty"`
Description string `json:"description,omitempty"`
Website string `json:"website,omitempty"`
CPE string `json:"cpe,omitempty"`
Icon string `json:"icon,omitempty"`
Cats []int `json:"cats,omitempty"`
CSS []string `json:"css,omitempty"`
DOM map[string]map[string]interface{} `json:"dom,omitempty"`
Cookies map[string]string `json:"cookies,omitempty"`
JS map[string]string `json:"js,omitempty"`
Headers map[string]string `json:"headers,omitempty"`
HTML []string `json:"html,omitempty"`
Script []string `json:"scripts,omitempty"`
ScriptSrc []string `json:"scriptSrc,omitempty"`
Meta map[string][]string `json:"meta,omitempty"`
Implies []string `json:"implies,omitempty"`
Description string `json:"description,omitempty"`
Website string `json:"website,omitempty"`
CPE string `json:"cpe,omitempty"`
Icon string `json:"icon,omitempty"`
}

const fingerprintURL = "https://raw.githubusercontent.com/enthec/webappanalyzer/main/src/technologies/%s.json"
Expand Down Expand Up @@ -155,7 +157,9 @@ func normalizeFingerprints(fingerprints *Fingerprints) *OutputFingerprints {
output := OutputFingerprint{
Cats: fingerprint.Cats,
Cookies: make(map[string]string),
DOM: make(map[string]map[string]interface{}),
Headers: make(map[string]string),
JS: make(map[string]string),
Meta: make(map[string][]string),
Description: fingerprint.Description,
Website: fingerprint.Website,
Expand All @@ -166,15 +170,36 @@ func normalizeFingerprints(fingerprints *Fingerprints) *OutputFingerprints {
for cookie, value := range fingerprint.Cookies {
output.Cookies[strings.ToLower(cookie)] = strings.ToLower(value)
}
for js := range fingerprint.JS {
output.JS = append(output.JS, strings.ToLower(js))
for k, v := range fingerprint.JS {
output.JS[k] = v
}
sort.Strings(output.JS)

for header, pattern := range fingerprint.Headers {
output.Headers[strings.ToLower(header)] = strings.ToLower(pattern)
}

// Use reflection for DOM as well
if fingerprint.Dom != nil {
v := reflect.ValueOf(fingerprint.Dom)

switch v.Kind() {
case reflect.String:
data := v.Interface().(string)
output.DOM[data] = map[string]interface{}{"exists": ""}
case reflect.Slice:
data := v.Interface().([]interface{})
for _, pattern := range data {
pat := pattern.(string)
output.DOM[pat] = map[string]interface{}{"exists": ""}
}
case reflect.Map:
data := v.Interface().(map[string]interface{})
for pattern, value := range data {
output.DOM[pattern] = value.(map[string]interface{})
}
}
}

// Use reflection type switch for determining HTML tag type
if fingerprint.HTML != nil {
v := reflect.ValueOf(fingerprint.HTML)
Expand Down
134 changes: 94 additions & 40 deletions fingerprints.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,21 @@ type Fingerprints struct {

// Fingerprint is a single piece of information about a tech validated and normalized
type Fingerprint struct {
Cats []int `json:"cats"`
CSS []string `json:"css"`
Cookies map[string]string `json:"cookies"`
JS []string `json:"js"`
Headers map[string]string `json:"headers"`
HTML []string `json:"html"`
Script []string `json:"scripts"`
ScriptSrc []string `json:"scriptSrc"`
Meta map[string][]string `json:"meta"`
Implies []string `json:"implies"`
Description string `json:"description"`
Website string `json:"website"`
CPE string `json:"cpe"`
Icon string `json:"icon"`
Cats []int `json:"cats"`
CSS []string `json:"css"`
Cookies map[string]string `json:"cookies"`
Dom map[string]map[string]interface{} `json:"dom"`
JS map[string]string `json:"js"`
Headers map[string]string `json:"headers"`
HTML []string `json:"html"`
Script []string `json:"scripts"`
ScriptSrc []string `json:"scriptSrc"`
Meta map[string][]string `json:"meta"`
Implies []string `json:"implies"`
Description string `json:"description"`
Website string `json:"website"`
CPE string `json:"cpe"`
Icon string `json:"icon"`
}

// CompiledFingerprints contains a map of fingerprints for tech detection
Expand All @@ -50,23 +51,33 @@ type CompiledFingerprint struct {
// icon contains a Icon associated with the fingerprint
icon string
// cookies contains fingerprints for target cookies
cookies map[string]*versionRegex
cookies map[string]*VersionRegex
// js contains fingerprints for the js file
js []*versionRegex
js map[string]*VersionRegex
// dom contains fingerprints for the target dom
dom map[string]map[string]*VersionRegex
// headers contains fingerprints for target headers
headers map[string]*versionRegex
headers map[string]*VersionRegex
// html contains fingerprints for the target HTML
html []*versionRegex
html []*VersionRegex
// script contains fingerprints for scripts
script []*versionRegex
script []*VersionRegex
// scriptSrc contains fingerprints for script srcs
scriptSrc []*versionRegex
scriptSrc []*VersionRegex
// meta contains fingerprints for meta tags
meta map[string][]*versionRegex
meta map[string][]*VersionRegex
// cpe contains the cpe for a fingerpritn
cpe string
}

func (f *CompiledFingerprint) GetJSRules() map[string]*VersionRegex {
return f.js
}

func (f *CompiledFingerprint) GetDOMRules() map[string]map[string]*VersionRegex {
return f.dom
}

// AppInfo contains basic information about an App.
type AppInfo struct {
Description string
Expand All @@ -81,17 +92,20 @@ type CatsInfo struct {
Cats []int
}

type versionRegex struct {
type VersionRegex struct {
regex *regexp.Regexp
skipRegex bool
group int
}

const versionPrefix = "version:\\"
const (
versionPrefix = "version:\\"
confidencePrefix = "confidence:"
)

// newVersionRegex creates a new version matching regex
// TODO: handles simple group cases only as of now (no ternary)
func newVersionRegex(value string) (*versionRegex, error) {
func newVersionRegex(value string) (*VersionRegex, error) {
splitted := strings.Split(value, "\\;")
if len(splitted) == 0 {
return nil, nil
Expand All @@ -102,8 +116,19 @@ func newVersionRegex(value string) (*versionRegex, error) {
return nil, err
}
skipRegex := splitted[0] == ""
regex := &versionRegex{regex: compiled, skipRegex: skipRegex}
regex := &VersionRegex{regex: compiled, skipRegex: skipRegex}
if skipRegex {
return regex, nil
}
for _, part := range splitted {
if strings.HasPrefix(part, confidencePrefix) {
confidence := strings.TrimPrefix(part, confidencePrefix)
if parsed, err := strconv.Atoi(confidence); err == nil {
if parsed < 10 { // Only use high confidence regex
return nil, nil
}
}
}
if strings.HasPrefix(part, versionPrefix) {
group := strings.TrimPrefix(part, versionPrefix)
if parsed, err := strconv.Atoi(group); err == nil {
Expand All @@ -116,7 +141,7 @@ func newVersionRegex(value string) (*versionRegex, error) {

// MatchString returns true if a version regex matched.
// The found version is also returned if any.
func (v *versionRegex) MatchString(value string) (bool, string) {
func (v *VersionRegex) MatchString(value string) (bool, string) {
if v.skipRegex {
return true, ""
}
Expand Down Expand Up @@ -155,16 +180,45 @@ func compileFingerprint(fingerprint *Fingerprint) *CompiledFingerprint {
description: fingerprint.Description,
website: fingerprint.Website,
icon: fingerprint.Icon,
cookies: make(map[string]*versionRegex),
js: make([]*versionRegex, 0, len(fingerprint.JS)),
headers: make(map[string]*versionRegex),
html: make([]*versionRegex, 0, len(fingerprint.HTML)),
script: make([]*versionRegex, 0, len(fingerprint.Script)),
scriptSrc: make([]*versionRegex, 0, len(fingerprint.ScriptSrc)),
meta: make(map[string][]*versionRegex),
dom: make(map[string]map[string]*VersionRegex),
cookies: make(map[string]*VersionRegex),
js: make(map[string]*VersionRegex),
headers: make(map[string]*VersionRegex),
html: make([]*VersionRegex, 0, len(fingerprint.HTML)),
script: make([]*VersionRegex, 0, len(fingerprint.Script)),
scriptSrc: make([]*VersionRegex, 0, len(fingerprint.ScriptSrc)),
meta: make(map[string][]*VersionRegex),
cpe: fingerprint.CPE,
}

for dom, patterns := range fingerprint.Dom {
compiled.dom[dom] = make(map[string]*VersionRegex)

for attr, value := range patterns {
switch attr {
case "exists", "text":
pattern, err := newVersionRegex(value.(string))
if err != nil {
continue
}
compiled.dom[dom]["main"] = pattern
case "attributes":
attrMap, ok := value.(map[string]interface{})
if !ok {
continue
}
compiled.dom[dom] = make(map[string]*VersionRegex)
for attrName, value := range attrMap {
pattern, err := newVersionRegex(value.(string))
if err != nil {
continue
}
compiled.dom[dom][attrName] = pattern
}
}
}
}

for header, pattern := range fingerprint.Cookies {
fingerprint, err := newVersionRegex(pattern)
if err != nil {
Expand All @@ -173,12 +227,12 @@ func compileFingerprint(fingerprint *Fingerprint) *CompiledFingerprint {
compiled.cookies[header] = fingerprint
}

for _, pattern := range fingerprint.JS {
for k, pattern := range fingerprint.JS {
fingerprint, err := newVersionRegex(pattern)
if err != nil {
continue
}
compiled.js = append(compiled.js, fingerprint)
compiled.js[k] = fingerprint
}

for header, pattern := range fingerprint.Headers {
Expand Down Expand Up @@ -214,7 +268,7 @@ func compileFingerprint(fingerprint *Fingerprint) *CompiledFingerprint {
}

for meta, patterns := range fingerprint.Meta {
var compiledList []*versionRegex
var compiledList []*VersionRegex

for _, pattern := range patterns {
fingerprint, err := newVersionRegex(pattern)
Expand Down Expand Up @@ -266,7 +320,7 @@ func (f *CompiledFingerprints) matchString(data string, part part) []string {
}

if version != "" {
app = formatAppVersion(app, version)
app = FormatAppVersion(app, version)
}
// Append the technologies as well as implied ones
technologies = append(technologies, app)
Expand Down Expand Up @@ -334,7 +388,7 @@ func (f *CompiledFingerprints) matchKeyValueString(key, value string, part part)

// Append the technologies as well as implied ones
if version != "" {
app = formatAppVersion(app, version)
app = FormatAppVersion(app, version)
}
technologies = append(technologies, app)
if len(fingerprint.implies) > 0 {
Expand Down Expand Up @@ -406,7 +460,7 @@ func (f *CompiledFingerprints) matchMapString(keyValue map[string]string, part p

// Append the technologies as well as implied ones
if version != "" {
app = formatAppVersion(app, version)
app = FormatAppVersion(app, version)
}
technologies = append(technologies, app)
if len(fingerprint.implies) > 0 {
Expand All @@ -417,7 +471,7 @@ func (f *CompiledFingerprints) matchMapString(keyValue map[string]string, part p
return technologies
}

func formatAppVersion(app, version string) string {
func FormatAppVersion(app, version string) string {
return fmt.Sprintf("%s:%s", app, version)
}

Expand Down

0 comments on commit c875ee8

Please sign in to comment.