Fix csv null and regen reports

Makefile

@@ -90,7 +90,7 @@ generate-testdata:
 	make install
 	go run ./hack/report/full.go
 	go run ./hack/samples/generate_samples.go
-	go run ./hack/report/full.go
+	go run ./hack/backport/backport.go
 
 
 

cmd/bundles/command.go

@@ -23,6 +23,8 @@ import (
 	"os/exec"
 	"strings"
 
+	"github.com/operator-framework/audit/pkg/actions"
+
 	"github.com/spf13/cobra"
 
 	// To allow create connection to query the index database
@@ -31,7 +33,6 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	"github.com/operator-framework/audit/pkg"
-	"github.com/operator-framework/audit/pkg/actions"
 	"github.com/operator-framework/audit/pkg/models"
 	index "github.com/operator-framework/audit/pkg/reports/bundles"
 )
@@ -202,38 +203,40 @@ func getDataFromIndexDB(report index.Data) (index.Data, error) {
 	defer row.Close()
 	for row.Next() {
 		var bundleName string
-		var csv string
+		var csv *string
 		var bundlePath string
 		var skipRange string
 		var version string
 		var replaces string
 		var skips string
 		var csvStruct *v1alpha1.ClusterServiceVersion
 
-		_ = row.Scan(&bundleName, &csv, &bundlePath, &version, &skipRange, &replaces, &skips)
+		err = row.Scan(&bundleName, &csv, &bundlePath, &version, &skipRange, &replaces, &skips)
+		if err != nil {
+			log.Errorf("unable to scan data from index %s\n", err.Error())
+		}
 
 		auditBundle := models.NewAuditBundle(bundleName, bundlePath)
-		err = json.Unmarshal([]byte(csv), &csvStruct)
-		if err == nil {
-			auditBundle.CSVFromIndexDB = csvStruct
-		} else {
-			auditBundle.Errors = append(auditBundle.Errors, fmt.Errorf("not found csv stored or"+
-				" unable to unmarshal data from the index.db: %s", err))
+
+		// the csv is pruned from the database to save space.
+		// See that is store only what is needed to populate the package manifest on cluster, all the extra
+		// manifests are pruned to save storage space
+		if csv != nil {
+			err = json.Unmarshal([]byte(*csv), &csvStruct)
+			if err == nil {
+				auditBundle.CSVFromIndexDB = csvStruct
+			} else {
+				auditBundle.Errors = append(auditBundle.Errors, fmt.Errorf("unable to parse the csv from the index.db: %s", err))
+			}
 		}
 
 		auditBundle.VersionDB = version
 		auditBundle.SkipRangeDB = skipRange
 		auditBundle.ReplacesDB = replaces
 		auditBundle.SkipsDB = skips
 
-		if len(bundlePath) > 0 {
-			// todo: improve the labels filter by implementing it in another way
-			auditBundle = actions.GetDataFromBundleImage(auditBundle, report.Flags.DisableScorecard,
-				report.Flags.DisableValidators, report.Flags.Label, report.Flags.LabelValue)
-		} else {
-			auditBundle.Errors = append(auditBundle.Errors,
-				errors.New("not found bundle path stored in the index.db"))
-		}
+		auditBundle = actions.GetDataFromBundleImage(auditBundle, report.Flags.DisableScorecard,
+			report.Flags.DisableValidators, report.Flags.Label, report.Flags.LabelValue)
 
 		sqlString := fmt.Sprintf("SELECT c.channel_name, c.package_name FROM channel_entry c "+
 			"where c.operatorbundle_name = '%s'", auditBundle.OperatorBundleName)

cmd/channels/command.go

@@ -16,8 +16,6 @@ package channels
 
 import (
 	"database/sql"
-	"encoding/json"
-	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -26,11 +24,9 @@ import (
 	log "github.com/sirupsen/logrus"
 	// To allow create connection to query the index database
 	_ "github.com/mattn/go-sqlite3"
-	"github.com/operator-framework/api/pkg/operators/v1alpha1"
 	"github.com/spf13/cobra"
 
 	"github.com/operator-framework/audit/pkg"
-	"github.com/operator-framework/audit/pkg/actions"
 	"github.com/operator-framework/audit/pkg/models"
 	"github.com/operator-framework/audit/pkg/reports/channels"
 )
@@ -176,18 +172,18 @@ func getDataFromIndexDB(report channels.Data) (channels.Data, error) {
 	for row.Next() {
 		var name string
 		var packageName string
-		var headOperatorBudle string
+		var headOperatorBundle string
 
-		if err := row.Scan(&name, &packageName, &headOperatorBudle); err != nil {
+		if err := row.Scan(&name, &packageName, &headOperatorBundle); err != nil {
 			log.Errorf("unable to scan data from index %s\n", err.Error())
 		}
 
-		col := models.NewAuditChannels(packageName, name, headOperatorBudle)
+		col := models.NewAuditChannels(packageName, name, headOperatorBundle)
 		report.AuditChannel = append(report.AuditChannel, *col)
 	}
 
 	for k, v := range report.AuditChannel {
-		sqlString := fmt.Sprintf("SELECT o.name, o.csv, o.bundlepath, o.version, o.skiprange, o.replaces, "+
+		sqlString := fmt.Sprintf("SELECT o.name, o.version, o.skiprange, o.replaces, "+
 			"o.skips from channel_entry ce, operatorbundle o "+
 			"WHERE ce.operatorbundle_name = o.name AND ce.channel_name = \"%s\"", v.ChannelName)
 
@@ -200,37 +196,21 @@ func getDataFromIndexDB(report channels.Data) (channels.Data, error) {
 
 		for row.Next() {
 			var bundleName string
-			var csv string
-			var bundlePath string
 			var skipRange string
 			var version string
 			var replaces string
 			var skips string
-			var csvStruct *v1alpha1.ClusterServiceVersion
 
-			_ = row.Scan(&bundleName, &csv, &bundlePath, &version, &skipRange, &replaces, &skips)
-
-			auditBundle := models.NewAuditBundle(bundleName, bundlePath)
-			err = json.Unmarshal([]byte(csv), &csvStruct)
-			if err == nil {
-				auditBundle.CSVFromIndexDB = csvStruct
-			} else {
-				auditBundle.Errors = append(auditBundle.Errors, errors.New("not found csv stored in the index.db"))
+			err = row.Scan(&bundleName, &version, &skipRange, &replaces, &skips)
+			if err != nil {
+				log.Errorf("unable to scan data from index %s\n", err.Error())
 			}
 
+			auditBundle := models.NewAuditBundle(bundleName, "")
 			auditBundle.VersionDB = version
 			auditBundle.SkipRangeDB = skipRange
 			auditBundle.ReplacesDB = replaces
 			auditBundle.SkipsDB = skips
-
-			// just check the bundle if we have not the csv from the db
-			if len(bundlePath) > 0 && auditBundle.CSVFromIndexDB == nil {
-				auditBundle = actions.GetDataFromBundleImage(auditBundle, true, true, "", "")
-			} else {
-				auditBundle.Errors = append(auditBundle.Errors,
-					errors.New("not found bundle path or csv stored in the index.db"))
-			}
-
 			report.AuditChannel[k].AuditBundles = append(report.AuditChannel[k].AuditBundles, *auditBundle)
 		}
 	}

cmd/packages/command.go

@@ -21,6 +21,8 @@ import (
 	"os/exec"
 	"strings"
 
+	"github.com/operator-framework/audit/pkg/actions"
+
 	log "github.com/sirupsen/logrus"
 
 	"database/sql"
@@ -32,7 +34,6 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/operator-framework/audit/pkg"
-	"github.com/operator-framework/audit/pkg/actions"
 	"github.com/operator-framework/audit/pkg/models"
 	"github.com/operator-framework/audit/pkg/reports/packages"
 )
@@ -250,28 +251,30 @@ func getDataFromIndexDB(report packages.Data) (packages.Data, error) {
 		defer row.Close()
 		for row.Next() {
 			var bundleName string
-			var csv string
+			var csv *string
 			var bundlePath string
 			var csvStruct *v1alpha1.ClusterServiceVersion
 			if err := row.Scan(&bundleName, &csv, &bundlePath); err != nil {
 				log.Errorf("unable to scan data from index %s\n", err.Error())
 			}
 
 			auditBundle := models.NewAuditBundle(bundleName, bundlePath)
-			err = json.Unmarshal([]byte(csv), csvStruct)
-			if err == nil {
-				auditBundle.CSVFromIndexDB = csvStruct
+			// the csv is pruned from the database to save space.
+			// See that is store only what is needed to populate the package manifest on cluster, all the extra
+			// manifests are pruned to save storage space
+			if csv != nil {
+				err = json.Unmarshal([]byte(*csv), &csvStruct)
+				if err == nil {
+					auditBundle.CSVFromIndexDB = csvStruct
+				} else {
+					auditBundle.Errors = append(auditBundle.Errors, fmt.Errorf("unable to parse the csv from the index.db: %s", err))
+				}
 			}
 
-			if len(bundlePath) > 0 {
-				// todo: improve labels rfe. That is done in this way just to address urgent need
-				auditBundle = actions.GetDataFromBundleImage(auditBundle,
-					report.Flags.DisableScorecard, report.Flags.DisableValidators,
-					report.Flags.Label, report.Flags.LabelValue)
-			} else {
-				auditBundle.Errors = append(auditBundle.Errors,
-					fmt.Errorf("not found bundle path for the bundle %s stored in the index.db", bundleName))
-			}
+			auditBundle = actions.GetDataFromBundleImage(auditBundle,
+				report.Flags.DisableScorecard, report.Flags.DisableValidators,
+				report.Flags.Label, report.Flags.LabelValue)
+
 			report.AuditPackage[k].AuditBundle = append(report.AuditPackage[k].AuditBundle, *auditBundle)
 		}
 	}

pkg/actions/get_bundle.go

@@ -16,6 +16,7 @@ package actions
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os/exec"
@@ -38,6 +39,12 @@ type Manifest struct {
 func GetDataFromBundleImage(auditBundle *models.AuditBundle,
 	disableScorecard, disableValidators bool, label, labelValue string) *models.AuditBundle {
 
+	if len(auditBundle.OperatorBundleImagePath) < 1 {
+		auditBundle.Errors = append(auditBundle.Errors,
+			errors.New("not found bundle path stored in the index.db"))
+		return auditBundle
+	}
+
 	downloadBundleImage(auditBundle)
 	bundleDir := createBundleDir(auditBundle)
 	extractBundleFromImage(auditBundle, bundleDir)

pkg/helpers.go

@@ -62,7 +62,7 @@ func RunCommand(cmd *exec.Cmd) ([]byte, error) {
 		return output, fmt.Errorf("%s failed with error: (%v) %s", command, err, string(output))
 	}
 	if len(output) > 0 {
-		log.Infof("command output :%s", output)
+		log.Debugf("command output :%s", output)
 	}
 	return output, nil
 }

pkg/reports/channels/data.go

@@ -20,9 +20,9 @@ import (
 	"sort"
 	"strings"
 
-	sq "github.com/Masterminds/squirrel"
-	"github.com/operator-framework/api/pkg/operators/v1alpha1"
+	"github.com/blang/semver"
 
+	sq "github.com/Masterminds/squirrel"
 	"github.com/operator-framework/audit/pkg"
 	"github.com/operator-framework/audit/pkg/models"
 	"github.com/operator-framework/audit/pkg/reports/bundles"
@@ -45,19 +45,43 @@ func (d *Data) PrepareReport() Report {
 
 		var allBundles []bundles.Columns
 		for _, v := range auditCha.AuditBundles {
-
 			bundles := bundles.Columns{}
+			bundles.Replace = v.ReplacesDB
+			bundles.SkipRange = v.SkipRangeDB
+			bundles.PackageName = v.PackageName
+			bundles.Channels = v.Channels
+			if len(v.SkipsDB) > 0 {
+				bundles.Skips = strings.Split(v.SkipsDB, ",")
+			}
 
-			var csv *v1alpha1.ClusterServiceVersion
-			if v.Bundle != nil && v.Bundle.CSV != nil {
-				csv = v.Bundle.CSV
-			} else if v.CSVFromIndexDB != nil {
-				csv = v.CSVFromIndexDB
+			if len(v.VersionDB) > 0 {
+				_, err := semver.Parse(v.VersionDB)
+				if err != nil {
+					bundles.InvalidVersioning = pkg.GetYesOrNo(true)
+				} else {
+					bundles.InvalidVersioning = pkg.GetYesOrNo(false)
+				}
 			}
 
-			bundles.AddDataFromCSV(csv)
-			bundles.AddDataFromBundle(v.Bundle)
-			allBundles = append(allBundles, bundles)
+			if len(v.SkipRangeDB) > 0 {
+				_, err := semver.ParseRange(v.SkipRangeDB)
+				if err != nil {
+					bundles.InvalidSkipRange = pkg.GetYesOrNo(true)
+				} else {
+					bundles.InvalidSkipRange = pkg.GetYesOrNo(false)
+				}
+			}
+
+			if len(v.ReplacesDB) > 0 {
+				// check if found replace
+				bundles.FoundReplace = pkg.GetYesOrNo(false)
+				for _, b := range auditCha.AuditBundles {
+					if b.OperatorBundleName == bundles.Replace {
+						bundles.FoundReplace = pkg.GetYesOrNo(true)
+						break
+					}
+				}
+			}
 		}
 
 		var auditErrors []error