feat(atarashi): Add Atarashi to FOSSology

.gitignore

@@ -27,6 +27,8 @@ install/defconf/Db.conf
 install/defconf/fossology.conf
 install/fo-postinstall
 src/adj2nest/agent/adj2nest
+src/atarashi/agent/atarashiWrapper
+src/atarashi/agent_tests/Functional/atarashiWrapper
 src/buckets/agent/buckets
 src/cli/cp2foss
 src/cli/cp2foss.1

install/scripts/install-atarashi.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e 
+
+sudo apt-get install software-properties-common
+sudo add-apt-repository ppa:deadsnakes/ppa
+sudo apt-get update
+sudo apt-get install python>=3.6
+
+curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+python3 get-pip.py
+
+pip install -i https://test.pypi.org/simple/ atarashi

src/Makefile

@@ -11,6 +11,7 @@ include $(VARS)
 DIRS = \
 	adj2nest \
 	buckets \
+	atarashi \
 	cli \
 	copyright \
 	debug \

src/atarashi/Makefile

@@ -0,0 +1,50 @@
+# Copyright Siemens AG 2019-2020
+#
+# Copying and distribution of this file, with or without modification,
+# are permitted in any medium without royalty provided the copyright
+# notice and this notice are preserved.  This file is offered as-is,
+# without any warranty.
+
+TOP = ../..
+VARS = $(TOP)/Makefile.conf
+include $(VARS)
+
+MOD_NAME = atarashiWrapper
+
+DIRS = agent ui
+TESTDIR = agent_tests
+
+DIR_LOOP = @set -e; for dir in $(DIRS); do $(MAKE) -s -C $$dir $(1); done
+
+all: VERSIONFILE
+	$(call DIR_LOOP, )
+
+test: all
+	$(TOP)/install/scripts/install-atarashi.sh
+	$(MAKE) -C $(TESTDIR) test
+
+coverage: all
+	$(MAKE) -C $(TESTDIR) coverage
+
+VERSIONFILE:
+	$(call WriteVERSIONFile,$(MOD_NAME))
+
+install: all
+	$(call DIR_LOOP,install)
+	$(INSTALL_DATA) VERSION $(DESTDIR)$(MODDIR)/$(MOD_NAME)/VERSION
+	$(INSTALL_DATA) $(MOD_NAME).conf $(DESTDIR)$(MODDIR)/$(MOD_NAME)/$(MOD_NAME).conf
+	mkdir -p $(DESTDIR)$(SYSCONFDIR)/mods-enabled
+	if test ! -e $(DESTDIR)$(SYSCONFDIR)/mods-enabled/$(MOD_NAME); then \
+		ln -s $(MODDIR)/$(MOD_NAME) $(DESTDIR)$(SYSCONFDIR)/mods-enabled; \
+	fi
+
+uninstall:
+	$(call DIR_LOOP,uninstall)
+	rm -rf $(DESTDIR)$(MODDIR)/$(MOD_NAME)
+	rm -f $(DESTDIR)$(SYSCONFDIR)/mods-enabled/$(MOD_NAME)
+
+clean:
+	$(call DIR_LOOP,clean)
+	rm -f VERSION
+
+.PHONY: all test coverage VERSIONFILE install uninstall clean

src/atarashi/README.md

@@ -0,0 +1,11 @@
+Introduction
+--------------
+
+Atarashi scans for license statements in open source software, focusing on text statistics. Designed to work stand-alone and with FOSSology. http://fossology.github.io/atarashi
+
+This FOSSology plugin is a wrapper for Atarashi. 
+
+Download
+--------------
+
+https://github.com/fossology/atarashi

src/atarashi/agent/Makefile

@@ -0,0 +1,66 @@
+# Copyright Siemens AG 2019-2020
+#
+# Copying and distribution of this file, with or without modification,
+# are permitted in any medium without royalty provided the copyright
+# notice and this notice are preserved.  This file is offered as-is,
+# without any warranty.
+
+TOP = ../../..
+VARS = $(TOP)/Makefile.conf
+include $(VARS)
+
+MOD_NAME = atarashiWrapper
+DEF = -DDATADIR='"$(MODDIR)"'
+CXXFLAGS_LOCAL = $(FO_CXXFLAGS) -I. -Wall -fopenmp $(shell pkg-config --cflags jsoncpp) 
+CXXFLAGS_LINK = $(FO_CXXLDFLAGS) -fopenmp $(shell pkg-config --libs jsoncpp)
+
+EXE = atarashiWrapper
+
+OBJECTS = databasehandler.o licensematch.o atarashi.o atarashiwrapper.o state.o utils.o
+COVERAGE = $(OBJECTS:%.o=%_cov.o)
+
+all: $(CXXFOLIB) $(EXE)
+
+$(EXE): $(CXXFOLIB) $(VARS) $(OBJECTS)
+	$(CXX) $(OBJECTS) $(DEF) $(CXXFLAGS_LINK) -o $@
+
+$(EXE)_cov: $(CXXFOLIB) $(VARS) $(COVERAGE)
+	$(CXX) $(COVERAGE) $(FLAG_COV) $(DEF) $(CXXFLAGS_LINK) -o $@
+
+#######################
+# library build rules #
+#######################
+
+$(CXXFOLIB):
+	$(MAKE) -C $(CXXFOLIBDIR)
+
+libatarashi.a: $(OBJECTS)
+	ar cvr $@ $(OBJECTS)
+
+libatarashi_cov.a: $(COVERAGE)
+	ar cvr $@ $(COVERAGE)
+
+######################
+# object build rules #
+######################
+
+$(OBJECTS): %.o: %.cc %.hpp
+	$(CXX) -c $(CXXFLAGS_LOCAL) $(DEF) $<
+
+$(COVERAGE): %_cov.o: %.cc %.hpp
+	$(CXX) -c $< $(CXXFLAGS_LOCAL) $(FLAG_COV) $(DEF) $(DEFS) -o $@
+
+#######################
+# install build rules #
+#######################
+
+install: $(EXE)
+	$(INSTALL_PROGRAM) $(EXE) $(DESTDIR)$(MODDIR)/$(MOD_NAME)/agent/$(EXE)
+
+uninstall:
+	rm -rf $(DESTDIR)$(MODDIR)/$(MOD_NAME)/$(EXE)/agent
+
+clean:
+	rm -f $(EXE) *.o *.a *.gcno *.gcda core
+
+.PHONY: all install uninstall clean

src/atarashi/agent/atarashi.cc

@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2019-2020, Siemens AG
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include "atarashi.hpp"
+
+using namespace fo;
+
+int main(int argc, char** argv)
+{
+  /* before parsing argv and argc make sure */
+  /* to initialize the scheduler connection */
+
+  DbManager dbManager(&argc, argv);
+  AtarashiDatabaseHandler databaseHandler(dbManager);
+
+  State state = getState(dbManager);
+
+  while (fo_scheduler_next() != NULL)
+  {
+    int uploadId = atoi(fo_scheduler_current());
+
+    if (uploadId == 0) continue;
+
+    int arsId = writeARS(state, 0, uploadId, 0, dbManager);
+
+    if (arsId <= 0)
+      bail(5);
+
+    if (!processUploadId(state, uploadId, databaseHandler))
+      bail(2);
+
+    fo_scheduler_heart(0);
+    writeARS(state, arsId, uploadId, 1, dbManager);
+  }
+  fo_scheduler_heart(0);
+
+  /* do not use bail, as it would prevent the destructors from running */
+  fo_scheduler_disconnect(0);
+  return 0;
+}

src/atarashi/agent/atarashi.hpp

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2019-2020, Siemens AG
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef ATARASHI_AGENT_ATARASHI_HPP
+#define ATARASHI_AGENT_ATARASHI_HPP
+
+#include "utils.hpp"
+
+extern "C" {
+#include "libfossagent.h"
+}
+
+#endif // ATARASHI_AGENT_ATARASHI_HPP

src/atarashi/agent/atarashiwrapper.cc

@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2019-2020, Siemens AG
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include <iostream>
+#include <boost/tokenizer.hpp>
+#include "atarashiwrapper.hpp"
+#include "utils.hpp"
+
+string scanFileWithAtarashi(const State& state, const fo::File& file)
+{
+  FILE* in;
+  char buffer[512];
+  string command = "atarashi -a DLD " + file.getFileName();   //Testing only for DLD Now
+  string result;
+
+  if (!(in = popen(command.c_str(), "r")))
+  {
+    cout << "could not execute atarashi command: " << command << endl;
+    bail(1);
+  }
+
+  while (fgets(buffer, sizeof(buffer), in) != NULL)
+  {
+    result += buffer;
+  }
+
+  if (pclose(in) != 0)
+  {
+    cout << "could not execute atarashi command: " << command << endl;
+    bail(1);
+  }
+
+  int jsonStart = result.find("{");
+  return result.substr(jsonStart, string::npos);
+}
+
+vector<LicenseMatch> extractLicensesFromAtarashiResult(string atarashiResult)
+{
+  Json::Reader reader;
+  Json::Value atarashiResultobj;
+  bool parseSuccess = reader.parse(atarashiResult, atarashiResultobj);
+
+  if (!parseSuccess)
+  {
+    cout << "Failed to parse" << reader.getFormattedErrorMessages();
+    bail(-30);
+  }
+
+  vector<LicenseMatch> matches;
+  Json::Value resultArray = atarashiResultobj["results"];
+  for (unsigned int index = 0; index < resultArray.size(); ++index)
+  {
+    Json::Value resultObject = resultArray[index];
+    LicenseMatch m(resultObject["shortname"].asString(),
+        (unsigned)(resultObject["sim_score"].asDouble() * 100.0));
+    matches.push_back(m);
+  }
+  return matches;
+}

src/atarashi/agent/atarashiwrapper.hpp

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2019-2020, Siemens AG
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef ATARASHI_AGENT_ATARASHI_WRAPPER_HPP
+#define ATARASHI_AGENT_ATARASHI_WRAPPER_HPP
+
+#include <string>
+#include <vector>
+#include <jsoncpp/json/json.h>
+
+#include "files.hpp"
+#include "licensematch.hpp"
+#include "state.hpp"
+
+using namespace std;
+
+string scanFileWithAtarashi(const State& state, const fo::File& file);
+vector<LicenseMatch> extractLicensesFromAtarashiResult(string atarashiResult);
+#endif // ATARASHI_AGENT_ATARASHI_WRAPPER_HPP