Skip to content

Commit

Permalink
refactor(fossdash UI menu) : created new menu and new php pages for f…
Browse files Browse the repository at this point in the history 
…ossdash.
  • Loading branch information
@darshank15
darshank15 committed Aug 9, 2020
1 parent aed5a17 commit 941ebbc
Show file tree
Hide file tree
Showing 8 changed files with 640 additions and 228 deletions.
20 changes: 20 additions & 0 deletions install/fossdash/config.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Example of fossdash metrics config file
QUERIES_NAME: [
"number_of_users", "number_of_groups", "number_of_file_uploads",
"number_of_projects__theoretically", "number_of_url_uploads",
"agents_count", "number_of_upload_status", "number_of_projects_per_size",
"reportgen_count", "pfile_count", "avg_pfile_count", "job_count"]

QUERY:
number_of_users: "SELECT count(u.*) AS users FROM users u;"
number_of_groups: "SELECT count(g.*) AS groups FROM groups g;"
number_of_projects__theoretically: "SELECT count(up.*) as uploads from (select distinct upload_mode, upload_origin from upload) up;"
number_of_file_uploads: "SELECT count(up1.upload_origin) as file_uploads FROM upload up1 WHERE up1.upload_mode = 104;"
number_of_url_uploads: "SELECT count(up2.upload_origin) as url_uploads FROM upload up2 WHERE up2.upload_mode = 100;"
agents_count: "SELECT ag.agent_name,count(jq.*) AS fired_jobs FROM agent ag LEFT OUTER JOIN jobqueue jq ON (jq.jq_type = ag.agent_name) GROUP BY ag.agent_name ORDER BY fired_jobs DESC;"
number_of_upload_status: "select CASE WHEN a.status=1 THEN 'open' WHEN a.status=2 THEN 'in_progres' WHEN a.status=3 THEN 'closed' WHEN a.status=3 THEN 'rejected' ELSE 'unknown' END status, a.count from (select status_fk status, count(1) as count from upload_clearing group by status_fk) a;"
number_of_projects_per_size: "select t.size, count(t.size) from (select CASE WHEN s.sx<2000 THEN 'small' WHEN s.sx>=2000 and s.sx<10000 THEN 'normal' ELSE 'big' END size from (select count(ut.ufile_name) sx from uploadtree ut group by upload_fk) s) t group by t.size;"
reportgen_count: "select count(*) from reportgen;"
pfile_count: "select count(*) from pfile;"
avg_pfile_count: "select COALESCE(round(avg(pfile_size)),0) from pfile;"
job_count: "select count(*) as jobs from job;"
44 changes: 22 additions & 22 deletions install/fossdash/fossdash-publish.py.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,19 +52,19 @@ config = " ".join(["=".join(config) for config in CONFIG_STATIC.items()])

timestamp = datetime.datetime.now().strftime("%s000000000")

# Quries to fetch sysconfig value from database
# Quries to fetch fossdashconfig value from database
QUERIES = \{
'uuid': "SELECT instance_uuid uuid FROM instance;",
'FossDashReportingAPIUrl': "select conf_value from sysconfig where variablename='FossDashReportingAPIUrl'",
'FossdashMetricConfig': "select conf_value from sysconfig where variablename='FossdashMetricConfig'",
'FossologyInstanceName': "select conf_value from sysconfig where variablename='FossologyInstanceName'",
'FossDashScriptCronSchedule': "select conf_value from sysconfig where variablename='FossDashScriptCronSchedule'",
'FossdashReportedCleaning': "select conf_value from sysconfig where variablename='FossdashReportedCleaning'",
'FossdashEnableDisable': "select conf_value from sysconfig where variablename='FossdashEnableDisable'",
'AuthType': "select conf_value from sysconfig where variablename='AuthType'",
'InfluxDBUser': "select conf_value from sysconfig where variablename='InfluxDBUser'",
'InfluxDBUserPassword': "select conf_value from sysconfig where variablename='InfluxDBUserPassword'",
'InfluxDBToken': "select conf_value from sysconfig where variablename='InfluxDBToken'"
'FossDashReportingAPIUrl': "select conf_value from fossdashconfig where variablename='FossDashReportingAPIUrl'",
'FossdashMetricConfig': "select conf_value from fossdashconfig where variablename='FossdashMetricConfig'",
'FossologyInstanceName': "select conf_value from fossdashconfig where variablename='FossologyInstanceName'",
'FossDashScriptCronSchedule': "select conf_value from fossdashconfig where variablename='FossDashScriptCronSchedule'",
'FossdashReportedCleaning': "select conf_value from fossdashconfig where variablename='FossdashReportedCleaning'",
'FossdashEnableDisable': "select conf_value from fossdashconfig where variablename='FossdashEnableDisable'",
'AuthType': "select conf_value from fossdashconfig where variablename='AuthType'",
'InfluxDBUser': "select conf_value from fossdashconfig where variablename='InfluxDBUser'",
'InfluxDBUserPassword': "select conf_value from fossdashconfig where variablename='InfluxDBUserPassword'",
'InfluxDBToken': "select conf_value from fossdashconfig where variablename='InfluxDBToken'"
\}

# prefix for generating fossdash fileName
Expand Down Expand Up @@ -319,11 +319,11 @@ def updateNewInstanceNameInAllFiles(new_instance_name):
print(error)


def getFossologyInstanceName(autogenerated_uuid,sysconfig_instance_name):
if sysconfig_instance_name is None:
def getFossologyInstanceName(autogenerated_uuid,fossdashconfig_instance_name):
if fossdashconfig_instance_name is None:
return autogenerated_uuid
else:
return sysconfig_instance_name
return fossdashconfig_instance_name

def disable_fossdash():
print("Disabling the fossdash !!!")
Expand Down Expand Up @@ -357,14 +357,6 @@ if __name__ == "__main__":
if fossology_instance_name_record is not None and fossology_instance_name_record[0] is not None:
fossology_instance_name = fossology_instance_name_record[0]

fossdash_metric_config_record = _query(connection, QUERIES['FossdashMetricConfig'], single=True)
fossdash_metric_config = None
if fossdash_metric_config_record is not None and fossdash_metric_config_record[0] is not None:
fossdash_metric_config = fossdash_metric_config_record[0]
else:
print("Missing fossdash metric config !!!")
exit(0)
fossdash_metric_yaml = yaml.load(fossdash_metric_config)
final_uuid = getFossologyInstanceName(autogenerated_uuid,fossology_instance_name)

agv_len = len(sys.argv)
Expand Down Expand Up @@ -392,6 +384,14 @@ if __name__ == "__main__":
print("Invalid command line argument")

print("Fossology Instance UUID = ",final_uuid)
fossdash_metric_config_record = _query(connection, QUERIES['FossdashMetricConfig'], single=True)
fossdash_metric_config = None
if fossdash_metric_config_record is not None and fossdash_metric_config_record[0] is not None:
fossdash_metric_config = fossdash_metric_config_record[0]
else:
print("Missing fossdash metric config !!!")
exit(0)
fossdash_metric_yaml = yaml.load(fossdash_metric_config)
final_metrics = getAllMetricsData(connection,final_uuid,fossdash_metric_yaml)
print("All data metrics Generated")
file_path = os.path.join(TEMP_BASE_DIR, file_name_prefix + "." + file_name_postfix_timestamp)
Expand Down
4 changes: 4 additions & 0 deletions install/fossinit.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,10 +133,14 @@ function explainUsage()
require_once("$MODDIR/lib/php/common-container.php");
require_once("$MODDIR/lib/php/common-cache.php");
require_once("$MODDIR/lib/php/common-sysconfig.php");
require_once("$MODDIR/lib/php/fossdash-config.php");

/* Initialize global system configuration variables $SysConfig[] */
ConfigInit($SYSCONFDIR, $SysConf);

/* Initialize fossdash configuration variables */
FossdashConfigInit($SYSCONFDIR, $SysConf);

/** delete from copyright where pfile_fk not in (select pfile_pk from pfile) */
/** add foreign constraint on copyright pfile_fk if not exist */
/** comment out for 2.5.0
Expand Down
154 changes: 7 additions & 147 deletions src/lib/php/common-sysconfig.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@
define("CONFIG_TYPE_PASSWORD", 4);
/** Dropdown type config */
define("CONFIG_TYPE_DROP", 5);
/** inputbox disabled type config */
define("CONFIG_TYPE_DISABLED", 6);


/**
* \brief Initialize the fossology system after bootstrap().
Expand Down Expand Up @@ -389,73 +388,6 @@ function Populate_sysconfig()
$valueArray[$variable] = array("'$variable'", "30", "'$patTokenValidityPrompt'",
strval(CONFIG_TYPE_INT), "'PAT'", "1", "'$patTokenValidityDesc'", "null", "null");

$variable = "FossdashEnableDisable";
$FossdashEnableDisablePrompt = _('Enable/Disable Fossdash');
$FossdashEnableDisableDesc = _('Start(Enable) or stop(Disable) the Fossdash');
$valueArray[$variable] = array("'$variable'", "'0'", "'$FossdashEnableDisablePrompt'",
strval(CONFIG_TYPE_DROP), "'FossDashAPI'", "1", "'$FossdashEnableDisableDesc'", "null", "'Disable{0}|Enable{1}'");

$variable = "FossDashReportingAPIUrl";
$fossdashApiUrlPrompt = _('FossDash Endpoint URL');
$URLValid = "check_url";
$fossdashApiUrlDesc = _('e.g. for Source Code : "http://localhost:8086/write?db=fossology_db" OR for Docker Setup : "http://influxdb:8086/write?db=fossology_db" .<br>Set the FossDash service endpoint. Disabled if empty.');
$valueArray[$variable] = array("'$variable'", "null", "'$fossdashApiUrlPrompt'",
strval(CONFIG_TYPE_TEXT), "'FossDashAPI'", "2", "'$fossdashApiUrlDesc'", "'$URLValid'", "null");

$variable = "FossdashMetricConfig";
$FossdashMetricConfigPrompt = _('Fossdash metric-reporting config');
$FossdashMetricConfigValid = "check_fossdash_config";
$FossdashMetricConfigDesc = _('modify the fossdash metric reporting config');
$valueArray[$variable] = array("'$variable'", "null", "'$FossdashMetricConfigPrompt'",
strval(CONFIG_TYPE_TEXTAREA), "'FossDashAPI'", "3", "'$FossdashMetricConfigDesc'", "'$FossdashMetricConfigValid'", "null");

$variable = "FossDashScriptCronSchedule";
$FossDashScriptCronSchedulePromt = _('cron job to run script');
$cronIntervalCheck= "check_cron_job_inteval";
$FossDashScriptCronScheduleDesc = _('Set the cron job of publishing script file for pushing data to time series db.');
$valueArray[$variable] = array("'$variable'", "'* * * * *'", "'$FossDashScriptCronSchedulePromt'",
strval(CONFIG_TYPE_TEXT), "'FossDashAPI'", "4", "'$FossDashScriptCronScheduleDesc'", "'$cronIntervalCheck'", "null");

$variable = "FossologyInstanceName";
$FossologyInstanceNamePrompt = _('Fosslogy instance name');
$instanceNameValid = "check_fossology_instance_name";
$FossologyInstanceNameDesc = _('e.g. Instance name formate = [a-zA-Z0-9_-]+ <br>Set the fossology instance name, leave empty to use autogenerated UUID value.');
$valueArray[$variable] = array("'$variable'", "null", "'$FossologyInstanceNamePrompt'",
strval(CONFIG_TYPE_TEXT), "'FossDashAPI'", "5", "'$FossologyInstanceNameDesc'", "'$instanceNameValid'", "null");

$variable = "FossdashReportedCleaning";
$FossdashReportingCleaningPrompt = _('Fossdash reported files cleaning');
$FossdashReportingCleaningValid = "check_fossdash_cleaning";
$FossdashReportingCleaningDesc = _('Number of days after which successfully pushed metrics are cleaned up. Set to Zero to disable cleanup.');
$valueArray[$variable] = array("'$variable'", "null", "'$FossdashReportingCleaningPrompt'",
strval(CONFIG_TYPE_TEXT), "'FossDashAPI'", "6", "'$FossdashReportingCleaningDesc'", "'$FossdashReportingCleaningValid'", "null");

$variable = "AuthType";
$AuthTypePrompt = _('Auth_type for InfluxDB');
$AuthTypeDesc = _('Select authentication type for an InfluxDB');
$valueArray[$variable] = array("'$variable'", "'0'", "'$AuthTypePrompt'",
strval(CONFIG_TYPE_DROP), "'FossDashAPI'", "7", "'$AuthTypeDesc'", "null", "'Token_based{0}|Uname_pass{1}'");

$variable = "InfluxDBUser";
$InfluxDBUserPrompt = _('InlfuxDB User');
$InfluxDBUserValid = "check_username";
$InfluxDBUserDesc = _('Set the username for InfluxDB.');
$valueArray[$variable] = array("'$variable'", "null", "'$InfluxDBUserPrompt'",
strval(CONFIG_TYPE_TEXT), "'FossDashAPI'", "8", "'$InfluxDBUserDesc'", "'$InfluxDBUserValid'", "null");

$variable = "InfluxDBUserPassword";
$InfluxDBUserPasswordPrompt = _('InlfuxDB Password');
$InfluxDBUserPasswordValid = "check_password";
$InfluxDBUserPasswordDesc = _('Set the password for Influx user. Password must atleast of lenght=3');
$valueArray[$variable] = array("'$variable'", "null", "'$InfluxDBUserPasswordPrompt'",
strval(CONFIG_TYPE_PASSWORD), "'FossDashAPI'", "9", "'$InfluxDBUserPasswordDesc'", "'$InfluxDBUserPasswordValid'", "null");

$variable = "InfluxDBToken";
$InfluxDBTokenPrompt = _('InlfuxDB Encoded Token');
$InfluxDBTokenDesc = _('Please Enter encoded token of your username for InfluxDB Authentication');
$valueArray[$variable] = array("'$variable'", "null", "'$InfluxDBTokenPrompt'",
strval(CONFIG_TYPE_TEXTAREA), "'FossDashAPI'", "10", "'$InfluxDBTokenDesc'", "null", "null");

$variable = "SkipFiles";
$mimeTypeToSkip = _("Skip MimeTypes from scanning");
$mimeTypeDesc = _("add comma (,) separated mimetype to exclude files from scanning");
Expand Down Expand Up @@ -501,7 +433,6 @@ function Populate_sysconfig()
}
unset($VarRec);
}

}

/**
Expand Down Expand Up @@ -704,10 +635,12 @@ function is_available($url, $timeout = 2, $tries = 2)
*/
function check_url($url)
{
if (filter_var($url, FILTER_VALIDATE_URL) && preg_match("#^((http)|(https)|(ftp)|(www)|(localhost))://(.*)#", $url) == 1) {
return 1;
} else {
if (empty($url) ||
preg_match("@^((http)|(https)|(ftp))://([[:alnum:]]+)@i", $url) != 1 ||
preg_match("@[[:space:]]@", $url) != 0) {
return 0;
} else {
return 1;
}
}

Expand All @@ -720,77 +653,4 @@ function check_IP($ip)
{
$e = "([0-9]|1[0-9]{2}|[1-9][0-9]|2[0-4][0-9]|25[0-5])";
return preg_match("/^$e\.$e\.$e\.$e$/", $ip);
}

/**
* \brief Check if the cron job schedule interval is valid
* \param string $cron_interval cron job interval
* \return 1: yes , 0: no
*/
function check_cron_job_inteval($cron_interval)
{
$cron_regex = "#^((@(annually|yearly|monthly|weekly|daily|hourly|reboot))|(@every (\d+(ns|us|µs|ms|s|m|h))+)|((((\d+,)+\d+|(\d+(\/|-)\d+)|\d+|\*|\*\/\d+) ?){5,7}))$#";
return preg_match($cron_regex, $cron_interval);
}


/**
* \brief Check if the fossology instance name is valid
* \param string $instance_name fossology instance name
* \return 1: yes , 0: no
*/
function check_fossology_instance_name($instance_name)
{
$instance_UUID_regex = "#^([a-zA-Z0-9_-]+)$#";
return preg_match($instance_UUID_regex, $instance_name);
}

/**
* \brief Check if cleaning_days is valid or not
* \param string $cleaning_days Number of days after which successfully pushed metrics are cleaned up
* \return 1: yes , 0: no
*/
function check_fossdash_cleaning($cleaning_days)
{
$numeric_day_regex = "#^[0-9][0-9]*$#";
return preg_match($numeric_day_regex, $cleaning_days);
}

/**
* \brief Check if given uname is valid or not
* \param string $uname username for influxDB
* \return 1: yes , 0: no
*/
function check_username($uname)
{
$uname_regex = "#^[A-Za-z0-9]+(?:[ _-][A-Za-z0-9]+)*$#";
return preg_match($uname_regex, $uname);
}

/**
* \brief Check if given password is valid or not
* \param string $uname password for influxDB
* \return 1: yes , 0: no
*/
function check_password($password)
{
$password_regex = "#^(?=.*[A-Za-z])[A-Za-z\d]{3,}$#";
return preg_match($password_regex, $password);
}

/**
* \brief Check if given config string does not contains any DB update or drop related commands
* \param string $config_str config for fossdash metrics
* \return 1: yes , 0: no
*/
function check_fossdash_config($config_str)
{
$lower_config_str = strtolower($config_str);
$db_update_command_list = array("drop", "insert", "update", "alter", "truncate", "delete");
foreach ($db_update_command_list as $cmd) {
if (strpos($lower_config_str,$cmd)) {
return 0;
}
}
return 1;
}
}
1 change: 1 addition & 0 deletions src/lib/php/common.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
require_once(dirname(dirname(dirname(__FILE__))) . "/vendor/autoload.php");

require_once("common-sysconfig.php");
require_once("fossdash-config.php");

// setup dependency injection
require_once("common-container.php");
Expand Down
Loading

0 comments on commit 941ebbc

Please sign in to comment.