Skip to content
Permalink
Browse files

Merge branch 'master' into master

  • Loading branch information...
fenugeek committed Aug 8, 2019
2 parents a2fe495 + 9e96012 commit 96f28fb85a0dceb44449c79dc0038115b254dbbd
Showing with 72 additions and 51 deletions.
  1. +21 −19 README.md
  2. +8 −7 dmarcts-report-parser.conf.sample
  3. +43 −25 dmarcts-report-parser.pl
@@ -41,36 +41,38 @@ git clone https://github.com/techsneeze/dmarcts-report-parser.git
or download a zip file containg all files from [here](https://github.com/techsneeze/dmarcts-report-parser/archive/master.zip). Once the files have been downloaded, you will need to copy/rename `dmarcts-report-parser.conf.sample` to `dmarcts-report-parser.conf`. Next, edit the configuration options:

```
####################################################################
### configuration ##################################################
####################################################################
################################################################################
### configuration ##############################################################
################################################################################
# If IMAP access is not used, config options starting with $imap
# do not need to be set and are ignored.
# If IMAP access is not used, config options starting with $imap do not need to
# be set and are ignored.
$debug = 0;
$delete_reports = 0;
$dbname = 'dmarc';
$dbuser = 'dmarc';
$dbpass = 'xxx';
$dbhost = ''; # Set the hostname if we can't connect to the local socket.
$imapserver = 'mail.example.com:143';
$imapuser = 'dmarcreports';
$imappass = 'xxx';
$imapssl = '0'; # If set to 1, remember to change server port to 993 and to disable imaptls.
$imaptls = '1'; # Enabled as the default and best-practice.
$tlsverify = '1'; # Enable verify server cert as the default and best-practice.
$imapignoreerror = 0; # set it to 1 if you see an "ERROR: message_string()
$dbpass = 'password';
$dbhost = 'dbhost'; # Set the hostname if we can't connect to the local socket.
$dbport = '3306';
$imapserver = 'imap.server';
$imapuser = 'username';
$imappass = 'password';
$imapport = '143';
$imapssl = '0'; # If set to 1, remember to change server port to 993 and disable imaptls.
$imaptls = '0'; # Enabled as the default and best-practice.
$tlsverify = '0'; # Enable verify server cert as the default and best-practice.
$imapignoreerror = '0'; # set it to 1 if you see an "ERROR: message_string()
# expected 119613 bytes but received 81873 you may
# need the IgnoreSizeErrors option" because of malfunction
# imap server as MS Exchange 2007, ...
$imapreadfolder = 'Inbox';
$imapreadfolder = 'dmarc';
# If $imapmovefolder is set, processed IMAP messages
# will be moved (overruled by the --delete option!)
$imapmovefolder = 'Inbox.processed';
# If $imapmovefolder is set, processed IMAP messages will be moved (overruled by
# the --delete option!)
$imapmovefolder = 'dmarc/processed';
# maximum size of XML files to store in database, long files can cause transaction aborts
$maxsize_xml = 50000;
@@ -10,25 +10,26 @@ $delete_reports = 0;

$dbname = 'dmarc';
$dbuser = 'dmarc';
$dbpass = 'xxx';
$dbhost = ''; # Set the hostname if we can't connect to the local socket.
$dbpass = 'password';
$dbhost = 'dbhost'; # Set the hostname if we can't connect to the local socket.
$dbport = '3306';

$imapserver = 'mail.example.com:143';
$imapuser = 'dmarcreports';
$imappass = 'xxx';
$imapserver = 'imap.server';
$imapuser = 'username';
$imappass = 'password';
$imapport = '143';
$imapssl = '0'; # If set to 1, remember to change server port to 993 and disable imaptls.
$imaptls = '1'; # Enabled as the default and best-practice.
$tlsverify = '1'; # Enable verify server cert as the default and best-practice.
$imapignoreerror = '0'; # set it to 1 if you see an "ERROR: message_string()
# expected 119613 bytes but received 81873 you may
# need the IgnoreSizeErrors option" because of malfunction
# imap server as MS Exchange 2007, ...
$imapreadfolder = 'Inbox';
$imapreadfolder = 'dmarc';

# If $imapmovefolder is set, processed IMAP messages will be moved (overruled by
# the --delete option!)
$imapmovefolder = 'Inbox.processed';
$imapmovefolder = 'dmarc/processed';

# If $imapmovefoldererr is set, IMAP messages that fail will be moved. If unset, failed messages
# will move to $imapmovefolder (if it is set). Overruled by the --delete option!
@@ -103,6 +103,7 @@ sub show_usage {
print " -r : Replace existing reports rather than skipping them. \n";
print " --delete : Delete processed message files (the XML is stored in the \n";
print " database for later reference). \n";
print " --info : Print out number of XML files or emails processed. \n";
print "\n";
}

@@ -117,8 +118,8 @@ sub show_usage {
# Define all possible configuration options.
our ($debug, $delete_reports, $delete_failed, $reports_replace, $maxsize_xml, $compress_xml,
$dbname, $dbuser, $dbpass, $dbhost, $dbport,
$imapserver, $imapuser, $imappass, $imapignoreerror, $imapssl, $imaptls, $imapmovefolder,
$imapmovefoldererr, $imapreadfolder, $imapopt, $tlsverify);
$imapserver, $imapport, $imapuser, $imappass, $imapignoreerror, $imapssl, $imaptls, $imapmovefolder,
$imapmovefoldererr, $imapreadfolder, $imapopt, $tlsverify, $processInfo);

# defaults
$maxsize_xml = 50000;
@@ -156,7 +157,7 @@ sub show_usage {
# Get command line options.
my %options = ();
use constant { TS_IMAP => 0, TS_MESSAGE_FILE => 1, TS_XML_FILE => 2, TS_MBOX_FILE => 3, TS_ZIP_FILE => 4 };
GetOptions( \%options, 'd', 'r', 'x', 'm', 'e', 'i', 'z', 'delete' );
GetOptions( \%options, 'd', 'r', 'x', 'm', 'e', 'i', 'z', 'delete', 'info' );

# Evaluate command line options
my $source_options = 0;
@@ -211,7 +212,7 @@ sub show_usage {
if (exists $options{r}) {$reports_replace = 1;}
if (exists $options{d}) {$debug = 1;}
if (exists $options{delete}) {$delete_reports = 1;}

if (exists $options{info}) {$processInfo = 1;}

# Setup connection to database server.
my $dbh = DBI->connect("DBI:mysql:database=$dbname;host=$dbhost;port=$dbport",
@@ -222,33 +223,46 @@ sub show_usage {

# Process messages based on $reports_source.
if ($reports_source == TS_IMAP) {
my $socketargs = '';
my $processedReport = 0;

# Disable verify mode for TLS support.
if ($imaptls == 1) {
if ( $tlsverify == 0 ) {
print "use tls without verify servercert.\n" if $debug;
$imapopt = [ SSL_verify_mode => SSL_VERIFY_NONE ];
} else {
print "use tls with verify servercert.\n" if $debug;
$imapopt = [ SSL_verify_mode => SSL_VERIFY_PEER ];
}
if ( $tlsverify == 0 ) {
print "use tls without verify servercert.\n" if $debug;
$imapopt = [ SSL_verify_mode => SSL_VERIFY_NONE ];
} else {
print "use tls with verify servercert.\n" if $debug;
$imapopt = [ SSL_verify_mode => SSL_VERIFY_PEER ];
}
# The whole point of setting this socket arg is so that we don't get the nasty warning
} else {
print "using ssl without verify servercert.\n" if $debug;
$socketargs = [ SSL_verify_mode => SSL_VERIFY_NONE ];
}


print "connection to $imapserver with Ssl => $imapssl, User => $imapuser, Ignoresizeerrors => $imapignoreerror\n" if $debug;

# Setup connection to IMAP server.
my $imap = Mail::IMAPClient->new( Server => $imapserver,
Ssl => $imapssl,
Starttls => $imapopt,
User => $imapuser,
Password => $imappass,
Ignoresizeerrors => $imapignoreerror,
Debug=> $debug
)
my $imap = Mail::IMAPClient->new(
Server => $imapserver,
Port => $imapport,
Ssl => $imapssl,
Starttls => $imapopt,
Debug => $debug,
Socketargs => $socketargs
)
# module uses eval, so we use $@ instead of $!
or die "IMAP Failure: $@";

# This connection is finished this way because of the tradgedy of exchange...
$imap->User($imapuser);
$imap->Password($imappass);
$imap->connect();

# Ignore Size Errors if we're using Exchange
$imap->Ignoresizeerrors($imapignoreerror);

# Set $imap to UID mode, which will force imap functions to use/return
# UIDs, instead of message sequence numbers. UIDs are not allowed to
# change during a session and are not allowed to be used twice. Looping
@@ -274,6 +288,7 @@ sub show_usage {
foreach my $msg (@msgs) {

my $processResult = processXML(TS_MESSAGE_FILE, $imap->message_string($msg), "IMAP message with UID #".$msg);
$processedReport++;
if ($processResult & 4) {
# processXML returned a value with database error bit enabled, do nothing at all!
next;
@@ -302,6 +317,7 @@ sub show_usage {

# We're all done with IMAP here.
$imap->logout();
if ( $debug || $processInfo ) { print "Processed $processedReport emails.\n"; }

} else { # TS_MESSAGE_FILE or TS_XML_FILE or TS_MBOX_FILE

@@ -372,7 +388,7 @@ sub show_usage {
}
}
}
print "Processed $counts messages(s).\n" if $debug;
if ($debug || $processInfo) { print "Processed $counts messages(s).\n"; }
}


@@ -409,15 +425,17 @@ sub moveToImapFolder {
}

sub processXML {
my $type = $_[0];
my $filecontent = $_[1];
my $f = $_[2];
my ($type, $filecontent, $f) = (@_);

if ($debug) {
print "\n";
print "----------------------------------------------------------------\n";
print "Processing $f \n";
print "----------------------------------------------------------------\n";
print "Type: $type\n";
print "FileContent: $filecontent\n";
print "MSG: $f\n";
print "----------------------------------------------------------------\n";
}

my $xml; #TS_XML_FILE or TS_MESSAGE_FILE
@@ -468,7 +486,7 @@ sub processXML {
# the fields of the first ZIPed XML file embedded into the message. The XML
# itself is not checked to be a valid DMARC report.
sub getXMLFromMessage {
my $message = $_[0];
my ($message) = (@_);

# fixup type in trustwave SEG mails
$message =~ s/ContentType:/Content-Type:/;

0 comments on commit 96f28fb

Please sign in to comment.
You can’t perform that action at this time.