Skip to content

Commit

Permalink
add feature for generating license meta data as json format (spdx2lio…
Browse files Browse the repository at this point in the history
…ssdb).
  • Loading branch information
tamada committed Oct 25, 2020
1 parent 8942c46 commit d1c5ca2
Show file tree
Hide file tree
Showing 5 changed files with 238 additions and 48 deletions.
183 changes: 141 additions & 42 deletions cmd/spdx2liossdb/main.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package main

import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
Expand All @@ -10,6 +11,7 @@ import (
flag "github.com/spf13/pflag"
"github.com/tamada/lioss"
"github.com/tamada/lioss/lib"
"gopkg.in/src-d/go-git.v4"
)

func helpMessage(prog string) string {
Expand All @@ -23,7 +25,9 @@ OPTIONS
-v, --verbose verbose mode.
-h, --help prints this message.
ARGUMENT
the directory contains SPDX license xml files.`, prog)
the directory contains SPDX license xml files.
NOTE
this is the internal command, and will not be distributed to the users.`, prog)
}

type cliOptions struct {
Expand All @@ -44,6 +48,11 @@ type runtimeOptions struct {
deprecated *withWithout
}

type LicenseData struct {
meta *lib.LicenseMeta
content string
}

func (ro *runtimeOptions) verbose(message string) {
if ro.verboseOpt {
fmt.Println(message)
Expand Down Expand Up @@ -92,61 +101,63 @@ func isTargetLicenseImpl(deprecated, osiApproved bool, meta *lib.LicenseMeta) bo
return !meta.Deprecated && !meta.OsiApproved
}

func readLicense(algo lioss.Algorithm, path string, opts *runtimeOptions) (*lioss.License, error) {
meta, licenseData, err := lib.ReadSPDX(path)
if err != nil {
return nil, err
}
if !isTargetLicense(opts, meta) {
return nil, nil
func generateLicense(algo lioss.Algorithm, data *LicenseData, opts *runtimeOptions) (*lioss.License, error) {
if !isTargetLicense(opts, data.meta) {
return nil, fmt.Errorf("%s: not target license", data.meta.Names.ShortName)
}
opts.verbosef("\t%s\n", meta.String())
return algo.Parse(strings.NewReader(licenseData), meta.Names.ShortName)
return algo.Parse(strings.NewReader(data.content), data.meta.Names.ShortName)
}

func appendLicensesIfNeeded(licenses []*lioss.License, license *lioss.License, err error) []*lioss.License {
if err == nil && license != nil {
licenses = append(licenses, license)
func performEachAlgorithm(db *lioss.Database, algo lioss.Algorithm, licenseData []*LicenseData, opts *runtimeOptions) error {
for _, data := range licenseData {
license, err := generateLicense(algo, data, opts)
if err != nil {
continue
}
db.Put(algo.String(), license)
}
return licenses
return nil
}

func readLicenses(algo lioss.Algorithm, target string, opts *runtimeOptions, infoList []os.FileInfo) []*lioss.License {
licenses := []*lioss.License{}
for _, info := range infoList {
if !info.IsDir() {
license, err := readLicense(algo, filepath.Join(target, info.Name()), opts)
licenses = appendLicensesIfNeeded(licenses, license, err)
}
func readLicenseDatum(target string, info os.FileInfo) (*LicenseData, error) {
if info.IsDir() {
return nil, fmt.Errorf("%s: is dir", info.Name())
}
meta, data, err := lib.ReadSPDX(filepath.Join(target, info.Name()))
if err != nil {
return nil, err
}
return licenses
return &LicenseData{meta: meta, content: data}, nil
}

func performEachAlgorithm(db *lioss.Database, algo lioss.Algorithm, target string, opts *runtimeOptions) error {
func readLicenseData(target string, opts *runtimeOptions) ([]*LicenseData, error) {
infoList, err := ioutil.ReadDir(target)
if err != nil {
return err
return nil, err
}
licenses := readLicenses(algo, target, opts, infoList)
for _, license := range licenses {
db.Put(algo.String(), license)
results := []*LicenseData{}
for _, info := range infoList {
result, err := readLicenseDatum(target, info)
if err == nil {
results = append(results, result)
}
}
return nil
return results, nil
}

func performEach(db *lioss.Database, algorithmName, target string, opts *runtimeOptions) error {
func performEach(db *lioss.Database, algorithmName string, licenseData []*LicenseData, opts *runtimeOptions) error {
algo, err := lioss.NewAlgorithm(algorithmName)
if err != nil {
return err
}
opts.verbose(algorithmName)
return performEachAlgorithm(db, algo, target, opts)
return performEachAlgorithm(db, algo, licenseData, opts)
}

func performImpl(db *lioss.Database, target string, opts *runtimeOptions) (int, error) {
func performImpl(db *lioss.Database, licenseData []*LicenseData, opts *runtimeOptions) (int, error) {
size := 0
for _, algorithmName := range lioss.AvailableAlgorithms {
err := performEach(db, algorithmName, target, opts)
err := performEach(db, algorithmName, licenseData, opts)
if err != nil {
return size, err
}
Expand All @@ -155,19 +166,97 @@ func performImpl(db *lioss.Database, target string, opts *runtimeOptions) (int,
return size, nil
}

func perform(dest, target string, opts *runtimeOptions) error {
fmt.Printf("read SPDX licenses %s-osi-approved, and %s-deprecated\n", opts.osiApproved.String(), opts.deprecated.String())
type generator interface {
Perform(licenseData []*LicenseData) error
}

type jsonGenerator struct {
dest string
from string
opts *runtimeOptions
}

type liossdbGenerator struct {
dest string
opts *runtimeOptions
}

func newGenerator(dest, from string, opts *runtimeOptions) generator {
if strings.HasSuffix(dest, ".json") {
return &jsonGenerator{dest: dest, from: from, opts: opts}
}
return &liossdbGenerator{dest: dest, opts: opts}
}

func readCommitID(dir string) (string, error) {
repository, err := git.PlainOpen(dir)
if err != nil {
return "", err
}
head, err := repository.Head()
if err != nil {
return "", err
}
return head.Hash().String(), nil
}

type jsonData struct {
Timestamp *lioss.Time `json:"timestamp"`
CommitID string `json:"git-commit-id"`
Licenses []*lib.LicenseMeta `json:"licenses"`
}

func (jg *jsonGenerator) Perform(data []*LicenseData) error {
id, _ := readCommitID(jg.from)
results := &jsonData{Timestamp: lioss.Now(), CommitID: id, Licenses: []*lib.LicenseMeta{}}
for _, datum := range data {
results.Licenses = append(results.Licenses, datum.meta)
}
return jg.writeImpl(results)
}

func (jg *jsonGenerator) writeImpl(results *jsonData) error {
writer, err := os.OpenFile(jg.dest, os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return err
}
defer writer.Close()
bytes, err := json.Marshal(results)
if err != nil {
return err
}
length, err := writer.Write(bytes)
if err != nil {
return err
}
if length != len(bytes) {
return fmt.Errorf("cannot write fully data, wont %d bytes, write %d bytes", len(bytes), length)
}
return nil
}

func (ldg *liossdbGenerator) Perform(data []*LicenseData) error {
fmt.Printf("read SPDX licenses %s-osi-approved, and %s-deprecated\n", ldg.opts.osiApproved.String(), ldg.opts.deprecated.String())
db := lioss.NewDatabase()
size, err := performImpl(db, target, opts)
size, err := performImpl(db, data, ldg.opts)
if err != nil {
return err
}
fmt.Printf("parse %d licenses for %d algorithms, and write database to %s...", size, len(db.Data), dest)
err2 := db.WriteTo(dest)
fmt.Printf("parse %d licenses for %d algorithms, and write database to %s...", size, len(db.Data), ldg.dest)
err2 := db.WriteTo(ldg.dest)
fmt.Println(" done")
return err2
}

func perform(dest, target string, opts *runtimeOptions) error {
licenseData, err := readLicenseData(target, opts)
if err != nil {
return err
}
generator := newGenerator(dest, target, opts)
return generator.Perform(licenseData)
}

func buildFlagSet(args []string) (*flag.FlagSet, *cliOptions) {
opts := new(cliOptions)
opts.runtimeOpts = &runtimeOptions{osiApproved: &withWithout{}, deprecated: &withWithout{}}
Expand All @@ -183,6 +272,19 @@ func buildFlagSet(args []string) (*flag.FlagSet, *cliOptions) {
return flags, opts
}

func validateWithAndWithout(dest string, opts *runtimeOptions) error {
if strings.HasSuffix(dest, ".json") {
return nil
}
if err := opts.deprecated.validate(); err != nil {
return fmt.Errorf("deprecated: %s", err.Error())
}
if err := opts.osiApproved.validate(); err != nil {
return fmt.Errorf("osi-approved: %s", err.Error())
}
return nil
}

func validateOptions(opts *cliOptions, flags *flag.FlagSet) (*cliOptions, error) {
if len(flags.Args()) <= 1 {
return nil, fmt.Errorf("no arguments specified")
Expand All @@ -191,11 +293,8 @@ func validateOptions(opts *cliOptions, flags *flag.FlagSet) (*cliOptions, error)
if len(realArgs) > 1 {
return nil, fmt.Errorf("arguments too much: %v", realArgs)
}
if err := opts.runtimeOpts.deprecated.validate(); err != nil {
return nil, fmt.Errorf("deprecated: %s", err.Error())
}
if err := opts.runtimeOpts.osiApproved.validate(); err != nil {
return nil, fmt.Errorf("osi-approved: %s", err.Error())
if err := validateWithAndWithout(opts.dest, opts.runtimeOpts); err != nil {
return nil, err
}
opts.target = realArgs[0]
return opts, nil
Expand Down
30 changes: 30 additions & 0 deletions cmd/spdx2liossdb/main_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package main

import (
"encoding/json"
"io/ioutil"
"os"
"sync"
"testing"
Expand All @@ -22,6 +24,8 @@ func Example_printHelp() {
// -h, --help prints this message.
// ARGUMENT
// the directory contains SPDX license xml files.
// NOTE
// this is the internal command, and will not be distributed to the users.
}

func TestGeneratedDataSize(t *testing.T) {
Expand Down Expand Up @@ -61,6 +65,32 @@ func testExec(t *testing.T, args []string, dest string, dataSize int, wg *sync.W
}
}

func TestJson(t *testing.T) {
goMain([]string{"spdx2liossdb", "-d", "test.json", "../../spdx/src"})
jsonData := &jsonData{}
reader, _ := os.Open("test.json")
defer reader.Close()
defer os.Remove("test.json")
data, _ := ioutil.ReadAll(reader)
if err := json.Unmarshal(data, jsonData); err != nil {
t.Errorf("%s", err.Error())
}
commitID := readString("../../.git/module/spdx/")
if jsonData.CommitID != commitID {
t.Errorf("commit id did not match, wont %s, got %s", commitID, jsonData.CommitID)
}
if len(jsonData.Licenses) != 409 {
t.Errorf("license length did not match, wont 409, got %d", len(jsonData.Licenses))
}
}

func readString(path string) string {
reader, _ := os.Open(path)
defer reader.Close()
data, _ := ioutil.ReadAll(reader)
return string(data)
}

func TestParseOptions(t *testing.T) {
testdata := []struct {
args []string
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,6 @@ require (
github.com/denisbrodbeck/striphtmltags v6.6.6+incompatible
github.com/h2non/filetype v1.0.12
github.com/spf13/pflag v1.0.5
gopkg.in/src-d/go-git.v4 v4.13.1
gopkg.in/xmlpath.v1 v1.0.0-20140413065638-a146725ea6e7
)
60 changes: 60 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,9 +1,69 @@
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/denisbrodbeck/striphtmltags v1.9.2 h1:oE5faQvhgwRke9VOPSZGmwlpiyBLDIykwzO2CQT+pB0=
github.com/denisbrodbeck/striphtmltags v6.6.6+incompatible h1:w4i4bsyWhAAqwUd9D/1NBi98citfaqCOI/8K3ZCh7KY=
github.com/denisbrodbeck/striphtmltags v6.6.6+incompatible/go.mod h1:wex3txg8OlzJKhtozM75/Ucy+jKUq73hqzl7XAcNeOY=
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/h2non/filetype v1.0.12 h1:yHCsIe0y2cvbDARtJhGBTD2ecvqMSTvlIcph9En/Zao=
github.com/h2non/filetype v1.0.12/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=
gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE=
gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8=
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
gopkg.in/xmlpath.v1 v1.0.0-20140413065638-a146725ea6e7 h1:zibSPXbkfB1Dwl76rJgLa68xcdHu42qmFTe6vAnU4wA=
gopkg.in/xmlpath.v1 v1.0.0-20140413065638-a146725ea6e7/go.mod h1:wo0SW5T6XqIKCCAge330Cd5sm+7VI6v85OrQHIk50KM=

0 comments on commit d1c5ca2

Please sign in to comment.