Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(cwe, cti): update dictionary #1553

Merged
merged 3 commits into from
Nov 1, 2022
Merged

Conversation

MaineK00n
Copy link
Collaborator

What did you implement:

update dictionary. (en: CWE v4.9, ja: 2022-10-31, MITRE ATT&CK: v12)

Type of change

  • New feature (non-breaking change which adds functionality)

How Has This Been Tested?

Checklist:

You don't have to satisfy all of the following.

  • Write tests
  • Write documentation
  • Check that there aren't other open pull requests for the same issue/feature
  • Format your source code by make fmt
  • Pass the test by make test
  • Provide verification config / commands
  • Enable "Allow edits from maintainers" for this PR
  • Update the messages below

Is this ready for review?: YES

Reference

@MaineK00n MaineK00n self-assigned this Oct 31, 2022
@MaineK00n
Copy link
Collaborator Author

for CWE en

package main

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
)

type weaknessCatalog struct {
	Weaknesses []weakness `xml:"Weaknesses>Weakness"`
}

type weakness struct {
	ID                  string `xml:"ID,attr"`
	Name                string `xml:"Name,attr"`
	Description         string `xml:"Description"`
	ExtendedDescription string `xml:"Extended_Description"`
}

func main() {
	if err := exec(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func exec() error {
	resp, err := http.Get("https://cwe.mitre.org/data/xml/cwec_latest.xml.zip")
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("failed to fetch https://cwe.mitre.org/data/xml/cwec_latest.xml.zip")
	}

	bs, err := io.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	r, err := zip.NewReader(bytes.NewReader(bs), int64(len(bs)))
	if err != nil {
		return err
	}

	f, err := r.File[0].Open()
	if err != nil {
		return err
	}
	defer f.Close()

	var catalog weaknessCatalog
	if err := xml.NewDecoder(f).Decode(&catalog); err != nil {
		return err
	}

	fmt.Println("// CweDictEn is the Cwe dictionary (https://cwe.mitre.org/data/xml/cwec_latest.xml.zip)")
	fmt.Println("var CweDictEn = map[string]Cwe{")
	for _, w := range catalog.Weaknesses {
		fmt.Printf("	%s: {\n", strconv.Quote(w.ID))
		fmt.Printf("		CweID:               %s,\n", strconv.Quote(w.ID))
		fmt.Printf("		Name:                %s,\n", strconv.Quote(w.Name))
		fmt.Printf("		Description:         %s,\n", strconv.Quote(strip(w.Description)))
		fmt.Printf("		ExtendedDescription: %s,\n", strconv.Quote(strip(w.ExtendedDescription)))
		fmt.Printf("		Lang:                %s,\n", strconv.Quote("en"))
		fmt.Println("	},")
	}
	fmt.Println("}")

	return nil
}

var rep = regexp.MustCompile(`\s{2,}`)

func strip(s string) string {
	return strings.TrimSpace(rep.ReplaceAllString(strings.NewReplacer("\t", " ", "\n", " ").Replace(s), " "))
}

@MaineK00n
Copy link
Collaborator Author

for CWE ja

package main

import (
	"encoding/xml"
	"fmt"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"golang.org/x/exp/maps"
	"golang.org/x/exp/slices"
)

type feed struct {
	Item []struct {
		References []struct {
			Source string `xml:"source,attr"`
			ID     string `xml:"id,attr"`
			Title  string `xml:"title,attr"`
		} `xml:"references"`
	} `xml:"item"`
}

func main() {
	if err := exec(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func exec() error {
	urls := []string{"https://jvndb.jvn.jp/ja/rss/jvndb.rdf", "https://jvndb.jvn.jp/ja/rss/jvndb_new.rdf"}
	for y := 2002; y <= time.Now().Year(); y++ {
		urls = append(urls, fmt.Sprintf("https://jvndb.jvn.jp/ja/rss/years/jvndb_%d.rdf", y))
	}

	cwes := map[string]string{}
	for _, u := range urls {
		if err := func() error {
			resp, err := http.Get(u)
			if err != nil {
				return err
			}
			defer resp.Body.Close()

			if resp.StatusCode != http.StatusOK {
				return fmt.Errorf("failed to fetch %s", u)
			}

			var f feed
			if err := xml.NewDecoder(resp.Body).Decode(&f); err != nil {
				return err
			}

			for _, i := range f.Item {
				for _, r := range i.References {
					if !strings.HasPrefix(r.ID, "CWE-") {
						continue
					}
					cwes[strings.TrimPrefix(r.ID, "CWE-")] = r.Title
				}
			}

			return nil
		}(); err != nil {
			return err
		}
	}

	fmt.Println("// CweDictJa is the Cwe dictionary")
	fmt.Println("var CweDictJa = map[string]Cwe{")
	ids := maps.Keys(cwes)
	slices.Sort(ids)
	for _, id := range ids {
		fmt.Printf("	%s: {\n", strconv.Quote(id))
		fmt.Printf("		CweID:               %s,\n", strconv.Quote(id))
		fmt.Printf("		Name:                %s,\n", strconv.Quote(cwes[id]))
		fmt.Printf("		Description:         %s,\n", strconv.Quote(""))
		fmt.Printf("		ExtendedDescription: %s,\n", strconv.Quote(""))
		fmt.Printf("		Lang:                %s,\n", strconv.Quote("ja"))
		fmt.Println("	},")

	}
	fmt.Println("}")

	return nil
}

@kotakanbe kotakanbe merged commit 8b5d1c8 into master Nov 1, 2022
@kotakanbe kotakanbe deleted the MaineK00n/update-cwe-cti branch November 1, 2022 05:00
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

Successfully merging this pull request may close these issues.

None yet

2 participants