Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added support for sending reports via HTTP(s) to multiple destinations #5158

Merged
merged 1 commit into from
Apr 29, 2024
Merged

Added support for sending reports via HTTP(s) to multiple destinations #5158

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
37 changes: 31 additions & 6 deletions Duplicati/Library/Modules/Builtin/ReportHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public abstract class ReportHelper : Interface.IGenericCallbackModule
/// <summary>
/// The default subject or title line
/// </summary>
protected virtual string DEFAULT_SUBJECT { get; }= "Duplicati %OPERATIONNAME% report for %backup-name%";
protected virtual string DEFAULT_SUBJECT { get; } = "Duplicati %OPERATIONNAME% report for %backup-name%";
/// <summary>
/// The default report level
/// </summary>
Expand Down Expand Up @@ -267,7 +267,8 @@ public void Configure(IDictionary<string, string> commandlineOptions)
var logLevel = Utility.Utility.ParseEnumOption(m_options, LogLevelOptionName, DEFAULT_LOG_LEVEL);

m_logstorage = new FileBackedStringList();
m_logscope = Logging.Log.StartScope(m => m_logstorage.Add(m.AsString(true)), m => {
m_logscope = Logging.Log.StartScope(m => m_logstorage.Add(m.AsString(true)), m =>
{

if (filter.Matches(m.FilterTag, out var result, out var match))
return result;
Expand Down Expand Up @@ -300,9 +301,33 @@ public virtual void OnStart(string operationname, ref string remoteurl, ref stri
/// <param name="exception">An optional exception that has stopped the backup</param>
/// <param name="subjectline">If set to <c>true</c>, the result is intended for a subject or title line.</param>
protected virtual string ReplaceTemplate(string input, object result, Exception exception, bool subjectline)
=> ReplaceTemplate(input, result, exception, subjectline, m_resultFormatSerializer);

/// <summary>
/// Helper method to perform template expansion
/// </summary>
/// <returns>The expanded template.</returns>
/// <param name="input">The input template.</param>
/// <param name="result">The result object.</param>
/// <param name="exception">An optional exception that has stopped the backup</param>
/// <param name="subjectline">If set to <c>true</c>, the result is intended for a subject or title line.</param>
/// <param name="format">The format to use when serializing the result</param>
protected virtual string ReplaceTemplate(string input, object result, Exception exception, bool subjectline, ResultExportFormat format)
=> ReplaceTemplate(input, result, exception, subjectline, ResultFormatSerializerProvider.GetSerializer(format));

/// <summary>
/// Helper method to perform template expansion
/// </summary>
/// <returns>The expanded template.</returns>
/// <param name="input">The input template.</param>
/// <param name="result">The result object.</param>
/// <param name="exception">An optional exception that has stopped the backup</param>
/// <param name="subjectline">If set to <c>true</c>, the result is intended for a subject or title line.</param>
/// <param name="resultFormatSerializer">The serializer to use when serializing the result</param>
protected virtual string ReplaceTemplate(string input, object result, Exception exception, bool subjectline, IResultFormatSerializer resultFormatSerializer)
{
// For JSON, ignore the template and just use the contents
if (ExportFormat == ResultExportFormat.Json && !subjectline)
if (resultFormatSerializer.Format == ResultExportFormat.Json && !subjectline)
{
var extra = new Dictionary<string, string>();

Expand Down Expand Up @@ -340,7 +365,7 @@ protected virtual string ReplaceTemplate(string input, object result, Exception
if (input.IndexOf($"%{kv.Key}%", StringComparison.OrdinalIgnoreCase) >= 0)
extra[kv.Key] = kv.Value;

return m_resultFormatSerializer.Serialize(result, exception, LogLines, extra);
return resultFormatSerializer.Serialize(result, exception, LogLines, extra);
}
else
{
Expand All @@ -356,7 +381,7 @@ protected virtual string ReplaceTemplate(string input, object result, Exception
else
{
if (input.IndexOf("%RESULT%", StringComparison.OrdinalIgnoreCase) >= 0)
input = Regex.Replace(input, "\\%RESULT\\%", m_resultFormatSerializer.Serialize(result, exception, LogLines, null), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
input = Regex.Replace(input, "\\%RESULT\\%", resultFormatSerializer.Serialize(result, exception, LogLines, null), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
}

foreach (KeyValuePair<string, string> kv in m_options)
Expand Down Expand Up @@ -446,7 +471,7 @@ public void OnFinish(object result, Exception exception)
}
catch (Exception ex)
{
Exception top = ex;
Exception top = ex;
var sb = new StringBuilder();
while (top != null)
{
Expand Down
216 changes: 137 additions & 79 deletions Duplicati/Library/Modules/Builtin/SendHttpMessage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,32 +19,49 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Interface;
using Duplicati.Library.Logging;
using Duplicati.Library.Modules.Builtin.ResultSerialization;
using Duplicati.Library.Utility;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace Duplicati.Library.Modules.Builtin {
namespace Duplicati.Library.Modules.Builtin
{
/// <summary>
/// Helper module to send HTTP report messages
/// </summary>
public class SendHttpMessage : ReportHelper
{
/// <summary>
/// The tag used for logging
/// </summary>
private static readonly string LOGTAG = Logging.Log.LogTagFromType<SendHttpMessage>();

/// <summary>
/// Entry describing a request to be sent
/// </summary>
/// <param name="Url">The url to send to</param>
/// <param name="Verb">The verb to use</param>
/// <param name="Format">The format to send</param>
private record SendRequestType(string Url, string Verb, ResultExportFormat Format);

#region Option names

/// <summary>
/// Option used to specify server URL
/// </summary>
private const string OPTION_URL = "send-http-url";
/// <summary>
/// Option used to specify server URLs for sending text reports
/// </summary>
private const string OPTION_URL_FORM = "send-http-form-urls";
/// <summary>
/// Option used to specify server URLs for sending json reports
/// </summary>
private const string OPTION_URL_JSON = "send-http-json-urls";
/// <summary>
/// Option used to specify report body
/// </summary>
private const string OPTION_MESSAGE = "send-http-message";
Expand Down Expand Up @@ -104,9 +121,9 @@ public class SendHttpMessage : ReportHelper

#region Private variables
/// <summary>
/// The HTTP report URL
/// The HTTP text report URLs
/// </summary>
private string m_url;
private List<SendRequestType> m_report_targets;
/// <summary>
/// The message parameter name
/// </summary>
Expand All @@ -115,10 +132,6 @@ public class SendHttpMessage : ReportHelper
/// The message parameter name
/// </summary>
private string m_extraParameters;
/// <summary>
/// The http verb
/// </summary>
private string m_verb;

#endregion

Expand All @@ -133,7 +146,7 @@ public class SendHttpMessage : ReportHelper
/// <summary>
/// A localized string describing the module with a friendly name
/// </summary>
public override string DisplayName { get { return Strings.SendHttpMessage.DisplayName;} }
public override string DisplayName { get { return Strings.SendHttpMessage.DisplayName; } }

/// <summary>
/// A localized description of the module
Expand Down Expand Up @@ -168,6 +181,9 @@ public override IList<ICommandLineArgument> SupportedCommands
new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.ReportHelper.OptionmaxloglinesShort, Strings.ReportHelper.OptionmaxloglinesLong, DEFAULT_LOGLINES.ToString()),

new CommandLineArgument(OPTION_RESULT_FORMAT, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.ResultFormatShort, Strings.ReportHelper.ResultFormatLong(Enum.GetNames(typeof(ResultExportFormat))), DEFAULT_EXPORT_FORMAT.ToString(), null, Enum.GetNames(typeof(ResultExportFormat))),

new CommandLineArgument(OPTION_URL_FORM, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpurlsformShort, Strings.SendHttpMessage.SendhttpurlsformLong),
new CommandLineArgument(OPTION_URL_JSON, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.SendhttpurlsjsonShort, Strings.SendHttpMessage.SendhttpurlsjsonLong),
});
}
}
Expand All @@ -181,100 +197,142 @@ public override IList<ICommandLineArgument> SupportedCommands
protected override string LogLinesOptionName => OPTION_MAX_LOG_LINES;
protected override string ResultFormatOptionName => OPTION_RESULT_FORMAT;

/// <summary>
/// This method is the interception where the module can interact with the execution environment and modify the settings.
/// </summary>
/// <param name="commandlineOptions">A set of commandline options passed to Duplicati</param>
protected override bool ConfigureModule(IDictionary<string, string> commandlineOptions)
/// <summary>
/// This method is the interception where the module can interact with the execution environment and modify the settings.
/// </summary>
/// <param name="commandlineOptions">A set of commandline options passed to Duplicati</param>
protected override bool ConfigureModule(IDictionary<string, string> commandlineOptions)
{
//We need a URL to report to
commandlineOptions.TryGetValue(OPTION_URL, out m_url);
if (string.IsNullOrEmpty(m_url))
var reportTargets = new List<SendRequestType>();

// Grab the legacy URL option if it exists, and add it to the appropriate list
commandlineOptions.TryGetValue(OPTION_URL, out var legacy_urls);
if (!string.IsNullOrEmpty(legacy_urls))
{
if (!commandlineOptions.TryGetValue(OPTION_RESULT_FORMAT, out var format))
format = ResultExportFormat.Duplicati.ToString();

if (!Enum.TryParse<ResultExportFormat>(format, out var exportFormat))
exportFormat = ResultExportFormat.Duplicati;

commandlineOptions.TryGetValue(OPTION_VERB, out var verb);
if (string.IsNullOrEmpty(verb))
verb = "POST";

reportTargets.AddRange(legacy_urls.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries).Select(url => new SendRequestType(url, verb, exportFormat)));
}

// Get the options as passed
commandlineOptions.TryGetValue(OPTION_URL_FORM, out var formurls);
if (!string.IsNullOrWhiteSpace(formurls))
reportTargets.AddRange(formurls.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries).Select(url => new SendRequestType(url, "POST", ResultExportFormat.Duplicati)));

commandlineOptions.TryGetValue(OPTION_URL_JSON, out var jsonurls);
if (!string.IsNullOrWhiteSpace(jsonurls))
reportTargets.AddRange(jsonurls.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries).Select(url => new SendRequestType(url, "POST", ResultExportFormat.Json)));

//We need at least one URL to report to
if (reportTargets.Count == 0)
return false;

m_report_targets = reportTargets;

commandlineOptions.TryGetValue(OPTION_MESSAGE_PARAMETER_NAME, out m_messageParameterName);
if (string.IsNullOrEmpty(m_messageParameterName))
m_messageParameterName = DEFAULT_MESSAGE_PARAMETER_NAME;

commandlineOptions.TryGetValue(OPTION_EXTRA_PARAMETERS, out m_extraParameters);
commandlineOptions.TryGetValue(OPTION_VERB, out m_verb);
if (string.IsNullOrWhiteSpace(m_verb))
m_verb = "POST";

return true;
}

#endregion

protected override string ReplaceTemplate(string input, object result, Exception exception, bool subjectline)
{
// No need to do the expansion as we throw away the result
if (subjectline)
return string.Empty;

return base.ReplaceTemplate(input, result, exception, subjectline);
}

protected override void SendMessage(string subject, string body) {
Exception ex = null;
#endregion

private async Task<Exception?> SendMessage(HttpClient client, SendRequestType target, string subject, string body)
{
byte[] data;
string contenttype;
MediaTypeHeaderValue contenttype;

if (ExportFormat == ResultExportFormat.Json)
if (target.Format == ResultExportFormat.Json)
{
contenttype = "application/json";
contenttype = new MediaTypeHeaderValue("application/json");
data = Encoding.UTF8.GetBytes(body);
}
else
{
contenttype = "application/x-www-form-urlencoded";
contenttype = new MediaTypeHeaderValue("application/x-www-form-urlencoded");
var postData = $"{m_messageParameterName}={System.Uri.EscapeDataString(body)}";
if (!string.IsNullOrEmpty(m_extraParameters))
{
postData += $"&{System.Uri.EscapeUriString(m_extraParameters)}";
}
data = Encoding.UTF8.GetBytes(postData);
}

var request = new HttpRequestMessage
{
RequestUri = new Uri(target.Url),
Method = new HttpMethod(target.Verb),
Content = new ByteArrayContent(data)
};
request.Content.Headers.ContentType = contenttype;

var request = (HttpWebRequest)WebRequest.Create(m_url);
request.ContentType = contenttype;
request.Method = m_verb;
request.ContentLength = data.Length;

try
try
{
var response = await client.SendAsync(request);
var responseContent = await response.Content.ReadAsStringAsync();

Logging.Log.WriteVerboseMessage(LOGTAG, "HttpResponseMessage",
"HTTP Response to {0}: {1} - {2}: {3}",
target.Url,
((int)response.StatusCode).ToString(),
response.ReasonPhrase,
responseContent
);
}
catch (Exception ex)
{
using (var stream = request.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}

using (var response = (HttpWebResponse)request.GetResponse())
{
Logging.Log.WriteVerboseMessage(LOGTAG,
"HttpResponseMessage",
"HTTP Response: {0} - {1}: {2}",
((int)response.StatusCode).ToString(),
response.StatusDescription,
new StreamReader(response.GetResponseStream()).ReadToEnd()
);
}
Logging.Log.WriteWarningMessage(LOGTAG, "HttpResponseError", ex, "HTTP Response request failed for: {0}", target.Url);
return ex;
}
catch (Exception e)

return null;
}

private Dictionary<ResultExportFormat, string> m_cachedBodyResults;
private string m_form_body = string.Empty;

protected override string ReplaceTemplate(string input, object result, Exception exception, bool subjectline)
{
// No need to do the expansion as we throw away the result
if (subjectline)
return string.Empty;

if (m_report_targets == null)
return string.Empty;


m_cachedBodyResults = m_report_targets
.Select(x => x.Format)
.Distinct()
.ToDictionary(
x => x,
x => base.ReplaceTemplate(input, result, exception, false, x)
);

return string.Empty;
}

protected override void SendMessage(string subject, string body)
{
if (m_report_targets == null || m_cachedBodyResults == null)
return;

using var client = new HttpClient();

Exception ex = null;

foreach (var target in m_report_targets)
{
ex = e;
if (ex is WebException exception && exception.Response is HttpWebResponse response)
{
Logging.Log.WriteWarningMessage(LOGTAG,
"HttpResponseError",
exception,
"HTTP Response: {0} - {1}: {2}",
((int)response.StatusCode).ToString(),
response.StatusDescription,
new StreamReader(response.GetResponseStream()).ReadToEnd()
);
}
if (m_cachedBodyResults.TryGetValue(target.Format, out var result))
ex ??= SendMessage(client, target, subject, body).ConfigureAwait(false).GetAwaiter().GetResult();
}

if (ex != null)
Expand Down