feat(ojo): New license scanner for SPDX-License-Identifier

The agent uses regex to find SPDX-License-Identifier and report all
the licenses found.
The agent can be used in stand alone mode as well.

Signed-off-by: Anupam Ghosh <anupam.ghosh@siemens.com>
Signed-off-by: Gaurav Mishra <mishra.gaurav@siemens.com>
Signed-off-by: Shaheem Azmal M MD <shaheem.azmal@siemens.com>

.gitignore

@@ -105,6 +105,7 @@ src/nomos/agent_tests/Functional/nomos-Xunit.xml
 src/nomos/agent_tests/Functional/nomos-regression-test.html
 src/nomos/agent_tests/Functional/report.d
 src/nomos/agent_tests/Unit/test_nomos
+src/ojo/agent/ojo
 src/pkgagent/agent/pkgagent
 src/scheduler/agent/defconf/init.d/fossology
 src/scheduler/agent/fo_cli

.travis.yml

@@ -22,6 +22,8 @@ addons:
       - genisoimage
       - libboost-program-options-dev
       - libboost-regex-dev
+      - libboost-system-dev
+      - libboost-filesystem-dev
       - libglib2.0-dev
       - libcppunit-dev
       - libcunit1-dev

Dockerfile

@@ -29,6 +29,7 @@ COPY ./src/copyright/mod_deps ./src/copyright/
 COPY ./src/delagent/mod_deps ./src/delagent/
 COPY ./src/mimetype/mod_deps ./src/mimetype/
 COPY ./src/nomos/mod_deps ./src/nomos/
+COPY ./src/ojo/mod_deps ./src/nomos/
 COPY ./src/pkgagent/mod_deps ./src/pkgagent/
 COPY ./src/scheduler/mod_deps ./src/scheduler/
 COPY ./src/ununpack/mod_deps ./src/ununpack/

debian/control

@@ -2,7 +2,7 @@ Source: fossology
 Section: utils
 Priority: extra
 Maintainer: Michael Jaeger <michael.c.jaeger@siemens.com>
-Build-Depends: debhelper, libglib2.0-dev, libmagic-dev, libxml2-dev, libmxml-dev, libtext-template-perl, librpm-dev, subversion, rpm, libpcre3-dev, libssl-dev, postgresql-server-dev-all, libboost-regex-dev, libboost-program-options-dev, libjsoncpp-dev, libjson-c-dev, php5-cli|php7.0-cli|php7.2-cli, php-mbstring|php5, php-zip|php5, php-xml|php5
+Build-Depends: debhelper, libglib2.0-dev, libmagic-dev, libxml2-dev, libmxml-dev, libtext-template-perl, librpm-dev, subversion, rpm, libpcre3-dev, libssl-dev, postgresql-server-dev-all, libboost-regex-dev, libboost-program-options-dev, libjsoncpp-dev, libjson-c-dev, php5-cli|php7.0-cli|php7.2-cli, php-mbstring|php5, php-zip|php5, php-xml|php5, libboost-system-dev, libboost-filesystem-dev
 Standards-Version: 3.9.1
 Homepage: http://fossology.org
 
@@ -236,6 +236,18 @@ Description: architecture for analyzing software, ninka
  .
  This package contains the ninka wrapper agent programs and their resources.
 
+Package: fossology-ojo
+Architecture: any
+Depends: fossology-common, ${shlibs:Depends}, ${misc:Depends}
+Description: architecture for analyzing software, ninka
+ The FOSSology project is a web based framework that allows you to
+ upload software to be picked apart and then analyzed by software agents
+ which produce results that are then browsable via the web interface.
+ Existing agents include license analysis, metadata extraction, and MIME
+ type identification.
+ .
+ This package contains the ojo agent programs and their resources.
+
 Package: fossology-decider
 Architecture: any
 Depends: fossology-common, ${misc:Depends}

debian/rules

@@ -156,6 +156,10 @@ endif
            PREFIX=/usr SYSCONFDIR=/etc/fossology LOCALSTATEDIR=/var \
            -C src/ninka install
 
+	$(MAKE) DESTDIR=$(CURDIR)/debian/fossology-ojo \
+           PREFIX=/usr SYSCONFDIR=/etc/fossology LOCALSTATEDIR=/var \
+           -C src/ojo install
+
 	$(MAKE) DESTDIR=$(CURDIR)/debian/fossology-decider \
            PREFIX=/usr SYSCONFDIR=/etc/fossology LOCALSTATEDIR=/var \
            -C src/decider install

src/Makefile

@@ -23,6 +23,7 @@ DIRS = \
 	monk \
 	ninka \
 	nomos \
+	ojo \
 	pkgagent \
 	readmeoss \
 	unifiedreport \

src/dox.c

@@ -168,6 +168,7 @@
  * -# \subpage maintagent
  * -# \subpage mimetype
  * -# \subpage nomos
+ * -# \subpage ojo
  * -# \subpage pkgagent
  * -# \subpage readmeoss
  * -# \subpage reuser

src/nomos/agent/json_writer.c

@@ -20,7 +20,7 @@
 #include "json_writer.h"
 #include "nomos.h"
 #include "nomos_utils.h"
-#include "json-c/json.h"
+#include <json-c/json.h>
 
 void writeToTemp()
 {

src/ojo/Makefile

@@ -0,0 +1,50 @@
+# Copyright Siemens AG 2019
+#
+# Copying and distribution of this file, with or without modification,
+# are permitted in any medium without royalty provided the copyright
+# notice and this notice are preserved.  This file is offered as-is,
+# without any warranty.
+
+TOP = ../..
+VARS = $(TOP)/Makefile.conf
+include $(VARS)
+
+MOD_NAME = ojo
+
+DIRS = agent ui
+TESTDIR = 
+
+DIR_LOOP = @set -e; for dir in $(DIRS); do $(MAKE) -s -C $$dir $(1); done
+
+all: VERSIONFILE ojo-all
+	$(call DIR_LOOP, )
+
+test: all
+	$(MAKE) -C $(TESTDIR) test
+
+coverage: all
+	$(MAKE) -C $(TESTDIR) coverage
+
+VERSIONFILE:
+	$(call WriteVERSIONFile,$(MOD_NAME))
+
+install: all
+	$(call DIR_LOOP,install)
+	$(INSTALL_DATA) VERSION $(DESTDIR)$(MODDIR)/$(MOD_NAME)/VERSION
+	$(INSTALL_DATA) $(MOD_NAME).conf $(DESTDIR)$(MODDIR)/$(MOD_NAME)/$(MOD_NAME).conf
+	mkdir -p $(DESTDIR)$(SYSCONFDIR)/mods-enabled
+	if test ! -e $(DESTDIR)$(SYSCONFDIR)/mods-enabled/$(MOD_NAME); then \
+		ln -s $(MODDIR)/$(MOD_NAME) $(DESTDIR)$(SYSCONFDIR)/mods-enabled; \
+	fi
+
+uninstall:
+	$(call DIR_LOOP,uninstall)
+	rm -rf $(DESTDIR)$(MODDIR)/$(MOD_NAME)
+	rm -f $(DESTDIR)$(SYSCONFDIR)/mods-enabled/$(MOD_NAME)
+
+clean:
+	$(call DIR_LOOP,clean)
+	rm -f VERSION
+
+.PHONY: all test coverage VERSIONFILE install uninstall clean
+.PHONY: ojo-all ojo-install ojo-uninstall ojo-clean ojo-Makefile

src/ojo/README.md

@@ -0,0 +1,8 @@
+Introduction
+--------------
+
+Ojo license identification agent, identifies the license(s) with the below format
+[SPDX-License-Identifier:  <"license name">]
+from text file under which source file is made available.
+
+

src/ojo/agent/Makefile

@@ -0,0 +1,75 @@
+# Copyright Siemens AG 2019
+#
+# Copying and distribution of this file, with or without modification,
+# are permitted in any medium without royalty provided the copyright
+# notice and this notice are preserved.  This file is offered as-is,
+# without any warranty.
+
+TOP = ../../..
+VARS = $(TOP)/Makefile.conf
+include $(VARS)
+
+DEF = -DDATADIR='"$(MODDIR)"'
+CXXFLAGS_LOCAL = $(FO_CXXFLAGS) -I. -Wall -fopenmp \
+                 $(shell pkg-config --cflags jsoncpp)
+
+CXXFLAGS_LINK = $(FO_CXXLDFLAGS) -fopenmp -lboost_regex -lboost_system \
+                -lboost_filesystem -lboost_program_options -lstdc++ \
+                $(shell pkg-config --libs jsoncpp)
+
+EXE = ojo
+
+OBJECTS = OjosDatabaseHandler.o OjoState.o OjoUtils.o directoryScan.o ojos.o
+COVERAGE = $(OBJECTS:%.o=%_cov.o)
+
+all: $(CXXFOLIB) $(EXE)
+
+$(EXE): $(CXXFOLIB) $(VARS) $(OBJECTS) OjoAgent.o ojoregex.hpp OjoState.hpp
+	$(CXX) $(OBJECTS) OjoAgent.o $(DEF) $(CXXFLAGS_LINK) -o $@
+
+$(EXE)_cov: $(CXXFOLIB) $(VARS) $(COVERAGE) OjoAgent_cov.o
+	$(CXX) $(COVERAGE) $(FLAG_COV) $(DEF) $(CXXFLAGS_LINK) -o $@
+
+#######################
+# library build rules #
+#######################
+
+libojo.a: $(OBJECTS) OjoAgent.o
+	ar cvr $@ $(OBJECTS) OjoAgent.o
+
+libojo_cov.a: $(COVERAGE) OjoAgent_cov.o
+	ar cvr $@ $(COVERAGE) OjoAgent_cov.o
+
+$(CXXFOLIB):
+	$(MAKE) -C $(CXXFOLIBDIR)
+
+######################
+# object build rules #
+######################
+
+$(OBJECTS): %.o: %.cc %.hpp
+	$(CXX) -c $(CXXFLAGS_LOCAL) $(DEF) $<
+
+OjoAgent.o: %.o: %.cc %.hpp ojoregex.hpp
+	$(CXX) -c $(CXXFLAGS_LOCAL) $(DEF) $<
+
+$(COVERAGE): %_cov.o: %.cc %.hpp
+	$(CXX) -c $< $(CXXFLAGS_LOCAL) $(FLAG_COV) $(DEF) $(DEFS) -o $@
+
+OjoAgent_cov.o: %_cov.o: %.cc %.hpp ojoregex.hpp
+	$(CXX) -c $(CXXFLAGS_LOCAL) $(DEF) $<
+
+#######################
+# install build rules #
+#######################
+
+install: $(EXE)
+	$(INSTALL_PROGRAM) $(EXE) $(DESTDIR)$(MODDIR)/$(EXE)/agent/$(EXE)
+
+uninstall:
+	rm -rf $(DESTDIR)$(MODDIR)/$(EXE)/agent
+
+clean:
+	rm -f $(EXE) *.o *.a *.gcno *.gcda core
+
+.PHONY: all install uninstall clean

src/ojo/agent/OjoAgent.cc

@@ -0,0 +1,165 @@
+/*
+ * Copyright (C) 2019, Siemens AG
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include "OjoAgent.hpp"
+
+using namespace std;
+
+/**
+ * Default constructor for OjoAgent.
+ *
+ * Also initializes the regex.
+ */
+OjoAgent::OjoAgent() :
+    regLicenseList(
+        boost::regex(SPDX_LICENSE_LIST, boost::regex_constants::icase)),
+    regLicenseName(
+        boost::regex(SPDX_LICENSE_NAMES, boost::regex_constants::icase))
+{
+}
+
+/**
+ * Scan a single file (when running from scheduler).
+ * @param filePath        The file to be scanned.
+ * @param databaseHandler Database handler to be used.
+ * @return List of matches found.
+ * @sa OjoAgent::scanString()
+ * @sa OjoAgent::filterMatches()
+ * @sa OjoAgent::findLicenseId()
+ * @throws std::runtime_error() Throws runtime error if the file can not be
+ * read with the file path in description.
+ */
+vector<ojomatch> OjoAgent::processFile(const string &filePath,
+  OjosDatabaseHandler &databaseHandler)
+{
+  ifstream stream(filePath);
+  std::stringstream sstr;
+  sstr << stream.rdbuf();
+  if (stream.fail())
+  {
+    throw std::runtime_error(filePath);
+  }
+  stream.close();
+  const string fileContent = sstr.str();
+  vector<ojomatch> licenseList;
+  vector<ojomatch> licenseNames;
+
+  scanString(fileContent, regLicenseList, licenseList, 0);
+  for (auto m : licenseList)
+  {
+    scanString(m.content, regLicenseName, licenseNames, m.start);
+  }
+
+  findLicenseId(licenseNames, databaseHandler);
+  filterMatches(licenseNames);
+
+  return licenseNames;
+}
+
+/**
+ * Scan a single file (when running from CLI).
+ *
+ * This function can not interact with DB.
+ * @param filePath File to be scanned
+ * @return List of matches.
+ */
+vector<ojomatch> OjoAgent::processFile(const string &filePath)
+{
+  ifstream stream(filePath);
+  std::stringstream sstr;
+  sstr << stream.rdbuf();
+  if (stream.fail())
+  {
+    throw std::runtime_error(filePath);
+  }
+  stream.close();
+  const string fileContent = sstr.str();
+  vector<ojomatch> licenseList;
+  vector<ojomatch> licenseNames;
+
+  scanString(fileContent, regLicenseList, licenseList, 0);
+  for (auto m : licenseList)
+  {
+    scanString(m.content, regLicenseName, licenseNames, m.start);
+  }
+
+  return licenseNames;
+}
+
+/**
+ * Scan a string based using a regex and create matches.
+ * @param text        String to be scanned
+ * @param reg         Regex to be used
+ * @param[out] result The match list.
+ * @param offset      The offset to be added for each match
+ */
+void OjoAgent::scanString(const string &text, boost::regex reg,
+    vector<ojomatch> &result, unsigned int offset)
+{
+  string::const_iterator end = text.end();
+  string::const_iterator pos = text.begin();
+
+  while (pos != end)
+  {
+    // Find next match
+    boost::smatch res;
+    if (boost::regex_search(pos, end, res, reg))
+    {
+      // Found match
+      result.push_back(
+        ojomatch(offset + res.position(1),
+          offset + res.position(1) + res.length(1),
+          res.length(1),
+          res[1].str()));
+      pos = res[0].second;
+      offset += res.position() + res.length();
+    }
+    else
+    {
+      // No match found
+      break;
+    }
+  }
+}
+
+/**
+ * Filter the matches list and remove entries with license id less than 1.
+ * @param[in,out] matches List of matches to be filtered
+ */
+void OjoAgent::filterMatches(vector<ojomatch> &matches)
+{
+  // Remvoe entries with license_fk < 1
+  matches.erase(
+    std::remove_if(matches.begin(), matches.end(), [](ojomatch match)
+    { return match.license_fk <= 0;}), matches.end());
+}
+
+/**
+ * Update the license id for each match entry
+ * @param[in,out] matches List of matches to be updated
+ * @param databaseHandler Database handler to be used
+ */
+void OjoAgent::findLicenseId(vector<ojomatch> &matches,
+  OjosDatabaseHandler &databaseHandler)
+{
+  // Update license_fk
+  for (size_t i = 0; i < matches.size(); ++i)
+  {
+    matches[i].license_fk = databaseHandler.getLicenseIdForName(
+      matches[i].content);
+  }
+}