feat(reports): Add new CSV report type

pbconf/fossology/rpm/fossology.spec

@@ -447,9 +447,11 @@ cp VERSION $RPM_BUILD_ROOT%{_sysconfdir}/PBPROJ/
 %{_sysconfdir}/PBPROJ/mods-enabled/spdx2
 %{_sysconfdir}/PBPROJ/mods-enabled/dep5
 %{_sysconfdir}/PBPROJ/mods-enabled/spdx2tv
+%{_sysconfdir}/PBPROJ/mods-enabled/spdx2csv
 %{_datadir}/PBPROJ/spdx2/*
 %{_datadir}/PBPROJ/dep5/*
 %{_datadir}/PBPROJ/spdx2tv/*
+%{_datadir}/PBPROJ/spdx2csv/*
 
 #
 # post

src/spdx2/Makefile

@@ -10,7 +10,7 @@ VARS = $(TOP)/Makefile.conf
 include $(VARS)
 
 MOD_NAME = spdx2
-MOD_NAMES = spdx2 spdx2tv dep5
+MOD_NAMES = spdx2 spdx2tv dep5 spdx2csv
 
 DIRS = agent ui
 TESTDIRS = agent_tests

src/spdx2/agent/Makefile

@@ -12,7 +12,7 @@ include $(VARS)
 
 MOD_NAME = spdx2
 MOD_SUBDIR = agent
-MOD_NAMES = spdx2 spdx2tv dep5
+MOD_NAMES = spdx2 spdx2tv dep5 spdx2csv
 
 EXE=spdx2.php version.php services.php spdx2utils.php
 

src/spdx2/agent/spdx2.php

@@ -15,6 +15,8 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, write to the Free Software Foundation, Inc.,
  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ * SPDX-License-Identifier: GPL-2.0
  */
 /**
  * @dir
@@ -85,7 +87,7 @@ class SpdxTwoAgent extends Agent
 
   const OUTPUT_FORMAT_KEY = "outputFormat";               ///< Argument key for output format
   const DEFAULT_OUTPUT_FORMAT = "spdx2";                  ///< Default output format
-  const AVAILABLE_OUTPUT_FORMATS = "spdx2,spdx2tv,dep5";  ///< Output formats available
+  const AVAILABLE_OUTPUT_FORMATS = "spdx2,spdx2tv,dep5,spdx2csv";  ///< Output formats available
   const UPLOAD_ADDS = "uploadsAdd";                       ///< Argument for additional uploads
 
   /** @var UploadDao $uploadDao
@@ -239,6 +241,8 @@ protected function getTemplateFile($partname)
         break;
       case "spdx2tv":
         break;
+      case "spdx2csv":
+        break;
       case "dep5":
         $prefix = $prefix . "copyright-";
         break;
@@ -264,6 +268,9 @@ protected function getFileBasename($packageName)
         case "spdx2tv":
           $fileName = $fileName .".spdx";
           break;
+        case "spdx2csv":
+          $fileName = $fileName .".csv";
+          break;
         case "dep5":
           $fileName = $fileName .".txt";
           break;

src/spdx2/agent/template/spdx2csv-document.twig

@@ -0,0 +1,8 @@
+{# Copyright 2021 Orange
+
+   Copying and distribution of this file, with or without modification,
+   are permitted in any medium without royalty provided the copyright notice and this notice are preserved.
+   This file is offered as-is, without any warranty.
+#}
+{{ packageNodes }}
+

src/spdx2/agent/template/spdx2csv-file.twig

@@ -0,0 +1,37 @@
+{# Copyright 2021 Orange
+
+   Copying and distribution of this file, with or without modification,
+   are permitted in any medium without royalty provided the copyright notice and this notice are preserved.
+   This file is offered as-is, without any warranty.
+#}
+{{- fileName -}},SPDXRef-item{{- fileId -}},
+{%- if isCleared -%}
+{%- if concludedLicenses|default is empty -%}
+NONE,
+{%- else -%}
+"{{- concludedLicense -}}",
+{%- endif -%}
+{%- else -%}
+NOASSERTION,
+{%- endif -%}
+{%- if licenseComment != null -%}
+"{{ licenseComment
+                                       |replace({ '\r\n': ' ', '\n': ' ', '\r': ' ' })
+                                       |replace({'\f':''})
+                                       |replace({'"':'""'}) }}"
+{%- endif -%},
+{%- if scannerLicenses|default is empty -%}
+NOASSERTION,
+{%- else -%}
+"{{- scannerLicenses|join('AND')
+                    |replace({' ':'-'})
+                    |replace({'AND':' AND '}) -}}",
+{%- endif -%}
+{%- if copyrights|default is empty -%}
+NOASSERTION
+{%- else -%}
+"{{- copyrights |join(';')
+                |replace({'<text>':'&lt;text&gt;','</text>':'&lt;/text&gt;'})
+                |replace({ '\r\n': ' ', '\n': ' ', '\r': ' ' })
+                |replace({'\f':''})
+                |replace({'"':'""'}) }}" {% endif %}                                    

src/spdx2/agent/template/spdx2csv-package.twig

@@ -0,0 +1,9 @@
+{# Copyright 2021 Orange
+
+   Copying and distribution of this file, with or without modification,
+   are permitted in any medium without royalty provided the copyright notice and this notice are preserved.
+   This file is offered as-is, without any warranty.
+#}
+FileName,SPDXID,LicenseConcluded,LicenseComment,LicenseInfoInFile,FileCopyrightText
+PackageName: {{ packageName  }} - PackageFileName: {{ uploadName }}
+{{ fileNodes }}

src/spdx2/spdx2csv.conf

@@ -0,0 +1,26 @@
+; Copyright 2021 Orange
+
+; Copying and distribution of this file, with or without modification,
+; are permitted in any medium without royalty provided the copyright
+; notice and this notice are preserved. This file is offered as-is,
+; without any warranty.
+
+; scheduler configure file for this agent
+[default]
+
+; name: The name of the agent from the agent table
+name = spdx2csv
+
+; command: The command that the scheduler will use when creating an instance of this agent. 
+; This will be parsed like a normal Unix command line.
+command = ../../spdx2/agent/spdx2 --outputFormat=spdx2csv
+
+; max: The maximum number of this agent that is allowed to exist at any one time. 
+; This is set to -1 if there is no limit on the number of instances of the agent.
+max = -1
+
+; special: Scheduler directive for special agent attributes.
+; A comma separated list of values.
+; Directives:
+;     EXCLUSIVE: the agent cannot run concurrently with any other agent. 
+special[] = 

src/spdx2/ui/Makefile

@@ -12,7 +12,7 @@ include $(VARS)
 MOD_NAME = spdx2
 MOD_SUBDIR = ui
 GENERAL_FILES = SpdxTwoAgentPlugin.php SpdxTwoGeneratorUi.php
-MOD_NAMES = spdx2 spdx2tv dep5
+MOD_NAMES = spdx2 spdx2tv dep5 spdx2csv
 
 all:
 	@echo "nothing to do"
@@ -27,6 +27,7 @@ install:
 	done
 	$(INSTALL_DATA) DepFiveAgentPlugin.php $(DESTDIR)$(MODDIR)/dep5/$(MOD_SUBDIR)
 	$(INSTALL_DATA) SpdxTwoTagValueAgentPlugin.php $(DESTDIR)$(MODDIR)/spdx2tv/$(MOD_SUBDIR)
+	$(INSTALL_DATA) SpdxTwoCommaSeparatedValuesAgentPlugin.php $(DESTDIR)$(MODDIR)/spdx2csv/$(MOD_SUBDIR)
 
 uninstall:
 	for MOD in $(MOD_NAMES); do \

src/spdx2/ui/SpdxTwoCommaSeparatedValuesAgentPlugin.php

@@ -0,0 +1,64 @@
+<?php
+/*
+ Copyright (C) 2021 Orange Author: Piotr Pszczola <piotr.pszczola@orange.com>
+
+ This program is free software; you can redistribute it and/or
+ modify it under the terms of the GNU General Public License
+ version 2 as published by the Free Software Foundation.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+ SPDX-License-Identifier: GPL-2.0
+
+ */
+
+namespace Fossology\SpdxTwo;
+
+use Fossology\Lib\Plugin\AgentPlugin;
+
+/**
+ * @class SpdxTwoCommaSeparatedValuesAgentPlugin
+ * @brief Add multiple uploads to CSV reports including SPDX identifiers
+ */
+class SpdxTwoCommaSeparatedValuesAgentPlugin extends AgentPlugin
+{
+  public function __construct()
+  {
+    $this->Name = "agent_spdx2csv";
+    $this->Title =  _("Export CSV report (SPDX)");
+    $this->AgentName = "spdx2csv";
+
+    parent::__construct();
+  }
+
+  /**
+   * @copydoc Fossology::Lib::Plugin::AgentPlugin::preInstall()
+   * @see Fossology::Lib::Plugin::AgentPlugin::preInstall()
+   */
+  function preInstall()
+  {
+    // no AgentCheckBox
+  }
+
+  /**
+   * @brief Add uploads to report
+   * @param array $uploads Array of upload ids
+   * @return string
+   */
+  public function uploadsAdd($uploads)
+  {
+    if (count($uploads) == 0) {
+      return '';
+    }
+    return '--uploadsAdd='. implode(',', array_keys($uploads));
+  }
+}
+
+register_plugin(new SpdxTwoCommaSeparatedValuesAgentPlugin());

src/spdx2/ui/SpdxTwoGeneratorUi.php

@@ -58,15 +58,18 @@ function __construct()
    */
   function preInstall()
   {
-    $text = _("Generate SPDX report");
-    menu_insert("Browse-Pfile::Export SPDX RDF", 0, self::NAME, $text);
+    $text = _("Generate SPDX report in RDF format");
+    menu_insert("Browse-Pfile::Export SPDX RDF report", 0, self::NAME, $text);
     menu_insert("UploadMulti::Generate SPDX", 0, self::NAME, $text);
 
     $text = _("Generate SPDX report in tag:value format");
-    menu_insert("Browse-Pfile::Export SPDX tag:value", 0, self::NAME . '&outputFormat=spdx2tv', $text);
+    menu_insert("Browse-Pfile::Export SPDX tag:value report", 0, self::NAME . '&outputFormat=spdx2tv', $text);
+
+    $text = _("Generate CSV report (with SPDX IDs)");
+    menu_insert("Browse-Pfile::Export SPDX CSV report (SPDX)", 0, self::NAME . '&outputFormat=spdx2csv', $text);
 
     $text = _("Generate Debian Copyright file");
-    menu_insert("Browse-Pfile::Export DEP5", 0, self::NAME . '&outputFormat=dep5', $text);
+    menu_insert("Browse-Pfile::Export DEP5 report", 0, self::NAME . '&outputFormat=dep5', $text);
   }
 
   /**

src/www/ui/async/AjaxShowJobs.php

@@ -291,6 +291,9 @@ protected function getShowJobsForEachJob($jobData)
           case 'spdx2tv':
             $jobArr['jobQueue'][$key]['download'] = "SPDX2 tag/value report";
             break;
+          case 'spdx2csv':
+            $varJobQueueRow['download'] = "SPDX2 CSV report";
+            break;
           case 'dep5':
             $jobArr['jobQueue'][$key]['download'] = "DEP5 copyright file";
             break;