Work in progress

techniques/system/common/1.0/common.st

@@ -12,6 +12,8 @@ bundle common system_common {
     "heartbeat_interval"    string => "&RUDDER_HEARTBEAT_INTERVAL&";
     "rudder_node_config_id" string => "&RUDDER_NODE_CONFIG_ID&";
     "agent_run_interval"    string => "&AGENT_RUN_INTERVAL&";
+    "rudder_syslog_port"     string => "&SYSLOGPORT&";
+    "rudder_syslog_protocol" string => "&RUDDER_SYSLOG_PROTOCOL&";
 
   classes:
 &if(INITIAL)&

techniques/system/common/1.0/metadata.xml

@@ -79,6 +79,12 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     <FILE name="properties.cf">
       <INCLUDED>true</INCLUDED>
     </FILE>
+    <FILE name="reporting-syslog.cf">
+      <INCLUDED>true</INCLUDED>
+    </FILE>
+    <FILE name="reporting-http.cf">
+      <INCLUDED>true</INCLUDED>
+    </FILE>
     <FILE name="rudder-agent-community-cron">
       <OUTPATH>common/cron/rudder-agent-community-cron</OUTPATH>
       <INCLUDED>false</INCLUDED>
@@ -125,6 +131,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     <NAME>RUDDER_SYSLOG_PROTOCOL</NAME>
     <NAME>RUDDER_NODE_GROUPS_VARS</NAME>
     <NAME>RUDDER_NODE_GROUPS_CLASSES</NAME>
+    <NAME>REPORTING_PROTOCOL</NAME>
   </SYSTEMVARS>
 
   <SECTIONS>

techniques/system/common/1.0/reporting-http.cf

@@ -0,0 +1,57 @@
+#####################################################################################
+# Copyright 2019 Normation SAS
+#####################################################################################
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, Version 3.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#####################################################################################
+
+# TODO purge too old reports
+
+bundle agent send_rudder_report(path) {
+  vars:
+      "endpoint" string => "https://${server_info.policy_server}/reports/";
+      "command"  string => "${g.rudder_curl} --tlsv1.2 --location --insecure --fail --silent --proxy '' --user ${g.davuser}:${g.davpw} --upload-file";
+
+  files:
+      "${path}" 
+        delete  => tidy,
+        if      => "send_report_${path}_ok"
+        classes => classes_generic("tidy_report_${path}");
+
+  commands:
+      "${command} ${path} ${endpoint}",
+        classes => classes_generic_two("send_report_${path}", "send_rudder_reports");
+
+  methods:
+      "any" usebundle => rudder_common_report("Common", "log_warn", "${system_common.directiveId}", "Log system for reports", "None", "Could not send report ${path}")
+                   if => "send_report_${path}_error";
+      # May be a another done by another agent run
+      "any" usebundle => rudder_common_report("Common", "log_warn", "${system_common.directiveId}", "Log system for reports", "None", "Could not remove report ${path}")
+                   if => "tidy_report_${path}_error";
+}
+
+bundle agent send_rudder_reports
+{
+  vars:
+      "raw_reports" slist => lsdir("${rudder_reports}/ready/", ".*", "false");
+      # Max 50 reports by run to avoid blocking the agent too long after a long
+      # disconnection
+      "reports"     slist => sublist("reports", "head", 50);
+
+  methods:
+      "send"    usebundle => send_rudder_report("${reports}")
+      "cleanup" usebundle => _classes_sanitize("send_rudder_reports");
+      "report"  usebundle => rudder_common_reports_generic("Common", "send_rudder_reports", "${system_common.directiveId}", "Log system for reports", "None", "Reports forwarding to policy server");
+}
+

techniques/system/common/1.0/reporting-syslog.cf

@@ -0,0 +1,297 @@
+#####################################################################################
+# Copyright 2011 Normation SAS
+#####################################################################################
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, Version 3.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#####################################################################################
+
+#######################################################
+# Check the log system, and configure it accordingly
+#
+# Note: if any configurations are changed here, corresponding services must be
+# restarted in the restart_services bundle
+
+bundle agent check_log_system
+{
+  vars:
+    debian::
+      "syslog_ng_source" string => "s_src";
+
+    SuSE::
+      "syslog_ng_source" string => "src";
+
+    redhat::
+      "syslog_ng_source" string => "s_sys";
+
+    use_tcp::
+      "rsyslog_rule_prefix"   string => "@@";
+      "syslogng_rule_prefix"  string => "tcp";
+    use_udp::
+      "rsyslog_rule_prefix"   string => "@";
+      "syslogng_rule_prefix"  string => "udp";
+
+    rsyslogd_conffile_present.rsyslog_version_5_6_4_ok::
+
+      "rsyslogd_message_reduction" string => "$RepeatedMsgReduction off${const.n}";
+
+    rsyslogd_conffile_present.rsyslog_version_5_6_4_not_ok::
+
+      "rsyslogd_message_reduction" string => "";
+
+    any::
+
+      "syslog_conf_comment"   string => "# Autogenerated by rudder, do not edit${const.n}";
+      "syslog_ng_conf_prefix" string => "filter f_local_rudder{facility(local6) and program(\"rudder.*|cf-agent.*\");};destination rudder_loghost {";
+      "syslog_ng_conf_proto"  string => "${syslogng_rule_prefix}";
+      "syslog_ng_conf_start"  string => "(\"";
+      "syslog_ng_conf_port"   string => "\" port (";
+      "syslog_ng_conf_portno" string => "${system_common.rudder_syslog_port}";
+      "syslog_ng_conf_suffix" string => "));};log {source(${syslog_ng_source});filter(f_local_rudder);destination(rudder_loghost);";
+      "syslog_ng_conf_final"  string => "flags(final);};";
+
+      "syslog_ng_conf"        string => concat("${syslog_conf_comment}", "${syslog_ng_conf_prefix}", "${syslog_ng_conf_proto}", "${syslog_ng_conf_start}", "${server_info.policy_server}", "${syslog_ng_conf_port}", "${syslog_ng_conf_portno}", "${syslog_ng_conf_suffix}", "${syslog_ng_conf_final}");
+      "syslog_ng_conf_regex"  string => concat(escape("filter f_local_rudder{facility(local6) and program("), ".*", escape(");};destination rudder_loghost {"), "(tcp|udp)", escape("${syslog_ng_conf_start}"), "[^\"]+", escape("${syslog_ng_conf_port}"), "[0-9]+", escape("${syslog_ng_conf_suffix}"), ".*");
+
+      "rsyslog_conffile"      string => "/etc/rsyslog.conf";
+      "syslogng_conffile"     string => "/etc/syslog-ng/syslog-ng.conf";
+      "syslog_conffile"       string => "/etc/syslog.conf";
+
+      "rsyslog_rudder_conffile" string => "/etc/rsyslog.d/rudder-agent.conf";
+
+  classes:
+
+    any::
+      "rsyslogd_conffile_present" expression => fileexists("${rsyslog_conffile}"), scope => "namespace";
+      "syslogng_conffile_present" expression => fileexists("${syslogng_conffile}"), scope => "namespace";
+      "syslogd_conffile_present"  expression => fileexists("${syslog_conffile}"), scope => "namespace";
+
+      "use_udp" expression => strcmp("${system_common.rudder_syslog_protocol}", "UDP");
+      "use_tcp" expression => strcmp("${system_common.rudder_syslog_protocol}", "TCP");
+
+      "pass3" expression => "pass2";
+      "pass2" expression => "pass1";
+      "pass1" expression => "any";
+
+      "check_rsyslog_version_present" expression => fileexists("${this.promise_dirname}/common/1.0/check-rsyslog-version");
+
+  files:
+
+    rsyslogd_conffile_present::
+      "${rsyslog_conffile}"
+        edit_line => append_if_no_lines("$IncludeConfig /etc/rsyslog.d/*.conf"),
+        edit_defaults => noempty_backup,
+        comment => "Add the rsyslog.conf.d include if not already present",
+        classes => kept_if_else("rsyslog_kept", "rsyslog_repaired" , "rsyslog_failed");
+
+    (rsyslog_version_5_6_4_ok|rsyslog_version_5_6_4_not_ok).rsyslogd_conffile_present.!policy_server::
+      "${rsyslog_rudder_conffile}"
+        edit_line => append_if_no_lines("#Rudder log system${const.n}${rsyslogd_message_reduction}${const.n}if $syslogfacility-text == 'local6' and $programname startswith 'rudder' then ${rsyslog_rule_prefix}${server_info.policy_server}:${system_common.rudder_syslog_port}${const.n}if $syslogfacility-text == 'local6' and $programname startswith 'rudder' then ~${const.n}if $syslogfacility-text == 'local6' and $programname startswith 'cf-agent' then ${rsyslog_rule_prefix}${server_info.policy_server}:${system_common.rudder_syslog_port}${const.n}if $syslogfacility-text == 'local6' and $programname startswith 'cf-agent' then ~"),
+        create => "true",
+        edit_defaults => empty_backup,
+        classes       => classes_generic("rsyslog");
+
+    SuSE.rsyslogd_conffile_present.policy_server::
+      # For SuSE, ensure that SYSLOG_DAEMON is set to 'rsyslogd' even if another syslog has been installed before
+      "/etc/sysconfig/syslog"
+        edit_line => ensure_rsyslogd_on_suse,
+        edit_defaults => noempty_backup,
+        classes       => classes_generic("rsyslog");
+
+    syslogng_conffile_present.!policy_server::
+      "${syslogng_conffile}"
+        edit_line => edit_syslog_conf_file("${syslog_ng_conf}", "${syslog_ng_conf_regex}"),
+        edit_defaults => noempty_backup,
+        classes       => classes_generic("syslog_ng");
+
+    syslogd_conffile_present.!policy_server::
+      "${syslog_conffile}"
+        edit_line => fix_syslogd("@${server_info.policy_server}"),
+        edit_defaults => noempty_backup,
+        classes       => classes_generic("syslogd");
+
+  methods:
+
+    pass2.syslogd_conffile_present::
+      "any" usebundle => rudder_common_report("Common", "log_info", "${system_common.directiveId}", "Log system for reports", "None", "Detected running syslog as syslogd");
+
+    pass2.syslogng_conffile_present::
+      "any" usebundle => rudder_common_report("Common", "log_info", "${system_common.directiveId}", "Log system for reports", "None", "Detected running syslog as syslog-ng");
+
+    pass2.rsyslogd_conffile_present::
+      "any" usebundle => rudder_common_report("Common", "log_info", "${system_common.directiveId}", "Log system for reports", "None", "Detected running syslog as rsyslog");
+
+    pass3.(syslogd_failed|syslog_ng_failed|rsyslog_failed)::
+      "any" usebundle => rudder_common_report("Common", "result_error", "${system_common.directiveId}", "Log system for reports", "None", "Logging system could not be configured for report centralization");
+
+    pass3.!(syslogd_failed|syslog_ng_failed|rsyslog_failed).(syslogd_repaired|syslog_ng_repaired|rsyslog_repaired)::
+      "any" usebundle => rudder_common_report("Common", "result_repaired", "${system_common.directiveId}", "Log system for reports", "None", "Configured logging system for report centralization");
+
+    pass3.!(syslogd_failed|syslog_ng_failed|rsyslog_failed).!(syslogd_repaired|syslog_ng_repaired|rsyslog_repaired).(syslogd_conffile_present.syslogd_kept|syslogng_conffile_present.syslog_ng_kept|rsyslogd_conffile_present.rsyslog_kept))::
+      "any" usebundle => rudder_common_report("Common", "result_success", "${system_common.directiveId}", "Log system for reports", "None", "Logging system for report centralization is already correctly configured");
+
+  commands:
+
+    rsyslogd_conffile_present.check_rsyslog_version_present::
+      # 5.6.4 is the first version mentionning RepeatedMsgReduction in http://www.rsyslog.com/change-log/
+      # The option is probably older though.
+      "/bin/sh ${this.promise_dirname}/common/1.0/check-rsyslog-version 5.6.4"
+        contain => in_shell,
+        module  => "true",
+        comment => "Check rsyslog version to know if it supports RepeatedMsgReduction";
+
+}
+
+#######################################################
+# Check the version of rsyslog, and correct the conf
+# file if > 5.7.1
+# This is done in another bundle than check_log_system
+# as it would make it too complex to read and maintain
+# (we would have needed to delay the restart of the services
+# at later iteration)
+#
+# Note: if any configurations are changed here, corresponding services must be
+# restarted in the restart_services bundle
+
+bundle agent check_rsyslog_version {
+
+  classes:
+      "check_rsyslog_version_present" expression => fileexists("${this.promise_dirname}/common/1.0/check-rsyslog-version");
+      "rsyslogd" expression => fileexists("${check_log_system.rsyslog_conffile}");
+
+    any::
+      "pass3" expression => "pass2";
+      "pass2" expression => "pass1";
+      "pass1" expression => "any";
+
+  files:
+    rsyslogd.rsyslog_version_5_7_1_ok::
+      "/etc/rsyslog.d/remove_limit.conf"
+        edit_line => append_if_no_lines("$SystemLogRateLimitInterval 0"),
+        edit_defaults => noempty_backup,
+        create    => "true",
+        comment   => "Add a config line in the rsyslog.conf file to prevent from dropping rudder messages",
+        classes   => classes_generic("rsyslog_limit");
+
+  methods:
+
+    pass3.(rsyslogd.!check_rsyslog_version_present)::
+      "any" usebundle => rudder_common_report("Common", "result_error", "${system_common.directiveId}", "Log system for reports", "None", "The file ${this.promise_dirname}/common/1.0/check-rsyslog-version is missing");
+
+    pass3.rsyslog_limit_error::
+      "any" usebundle => rudder_common_report("Common", "result_error", "${system_common.directiveId}", "Log system for reports", "None", "Could not remove message limit in rsyslog");
+
+    pass3.rsyslog_limit_repaired::
+      "any" usebundle => rudder_common_report("Common", "log_repaired", "${system_common.directiveId}", "Log system for reports", "None", "Updated the rsyslog configuration to remove message limit");
+
+  commands:
+    rsyslogd.check_rsyslog_version_present::
+      "/bin/sh ${this.promise_dirname}/common/1.0/check-rsyslog-version"
+        contain => in_shell,
+        module  => "true",
+        comment => "Check rsyslog version in order to add or not a configuration line in rsyslog.conf";
+
+}
+
+#############################################################
+# Remove syslog configurations that Rudder may have applied #
+#############################################################
+# Note: if any configurations are changed here, corresponding services must be
+# restarted in the restart_services bundle
+
+bundle agent remove_rudder_syslog_configuration {
+  vars:
+      "syslogng_delete_lines_patterns" slist => {
+        "\s*# Autogenerated by rudder, do not edit.*",
+        ".*destination\(rudder_loghost\);.*",
+      };
+
+      "syslog_delete_lines_patterns" slist => {
+        "\s*# Rudder specific logging parameters.*",
+        "local6.notice\s+@${server_info.policy_server}",
+      };
+
+  classes:
+      "syslogng" expression => fileexists("${check_log_system.syslogng_conffile}");
+      "syslogd"  expression => fileexists("${check_log_system.syslog_conffile}");
+
+  files:
+      "${check_log_system.rsyslog_rudder_conffile}"
+        delete         => tidy,
+        classes        => classes_generic("remove_rudder_syslog_configuration_rsyslog");
+
+    syslogng::
+      "${check_log_system.syslogng_conffile}"
+        edit_line      => delete_lines_matching("${syslogng_delete_lines_patterns}"),
+        edit_defaults  => noempty_backup,
+        classes        => classes_generic("remove_rudder_syslog_configuration_syslogng");
+
+    syslogd::
+      "${check_log_system.syslog_conffile}"
+        edit_line      => delete_lines_matching("${syslog_delete_lines_patterns}"),
+        edit_defaults  => noempty_backup,
+        classes        => classes_generic("remove_rudder_syslog_configuration_syslog");
+
+  methods:
+      "merge result classes" usebundle  => _classes_combine_three("remove_rudder_syslog_configuration_rsyslog", "remove_rudder_syslog_configuration_syslogng", "remove_rudder_syslog_configuration_syslog", "remove_rudder_syslog_configuration_result");
+      "reports"              usebundle  => rudder_common_reports_generic("Common", "remove_rudder_syslog_configuration_result", "${system_common.directiveId}", "Log system for reports", "None", "Rudder syslog configuration removed");
+
+}
+
+# Fix syslogd content : caution, the @ must be in the argument
+bundle edit_line fix_syslogd(syslogd)
+{
+  delete_lines:
+      "^\s*local6.*"
+        comment => "Delete all local6 facility related lines";
+
+  insert_lines:
+      "# Rudder specific logging parameters";
+      "local6.notice ${syslogd}"
+        comment => "Add the rudder syslogd destination";
+}
+
+bundle edit_line edit_syslog_conf_file(line_to_add, pattern_to_remove)
+{
+  delete_lines:
+      "${pattern_to_remove}";
+
+      # Remove any unexpanded variables that may have slipped through due to old bugs or race conditions at installation time
+      "\$\(syslog_ng_conf\)";
+      "\$\{syslog_ng_conf\}";
+
+      # Remove the syslog config when the loghost was just called "loghost" (now called "rudder_loghost")
+      # This was used in Rudder pre-3.0. We need to keep this in all versions that support upgrading from pre-3.0 versions (up to and including 2.11)
+      # See http://www.rudder-project.org/redmine/issues/5773
+      "filter\ f\_local\_rudder\{facility\(local6\)\ and\ program\(\"rudder\.\*\"\)\;\}\;destination\ loghost\ \{(tcp|udp)\(\"[^\"]+\".*";
+
+  insert_lines:
+      "${line_to_add}"
+        location => syslogng_log_part;
+}
+
+body location syslogng_log_part
+{
+        select_line_matching => "^\s*log\s*\{.*";
+        before_after => "before";
+        first_last => "first";
+}
+
+bundle edit_line ensure_rsyslogd_on_suse
+{
+  field_edits:
+      # match a line starting like 'SYSLOG_DAEMON=something'
+      "^SYSLOG_DAEMON=.*$"
+        edit_field => col("=","2","\"rsyslogd\"","set"),
+        comment => "Match a line starting like key = something";
+}

techniques/system/common/1.0/site.cf

@@ -46,6 +46,7 @@ bundle common g
       "rudder_var_tmp"             string => "${rudder_var}/tmp"; # tmp generated data
       "rudder_base_sbin"           string => "${rudder_base}/sbin"; #folder where tools are installed
       "rudder_inventories"         string => "${rudder_var}/inventories";
+      "rudder_reports"             string => "${rudder_var}/reports";
       "rudder_disable_agent_file"  string => "${rudder_base}/etc/disable-agent";
       "rudder_disable_server_file" string => "${rudder_base}/etc/disable-policy-distribution";
       "rudder_disable_inventory_file" string => "${rudder_base}/etc/disable-inventory";