Skip to content

Commit

Permalink
Merge pull request #4884 from cyrusimap/jmap_email_import_receivedat
Browse files Browse the repository at this point in the history 
JMAP Email: update Email/import logic for receivedAt property
  • Loading branch information
@rsto
rsto committed Apr 15, 2024
2 parents 1bca21b + ed5f407 commit 19961c5
Show file tree
Hide file tree
Showing 4 changed files with 111 additions and 51 deletions.
86 changes: 53 additions & 33 deletions cassandane/tiny-tests/JMAPEmail/email_import_received_at
View file
Expand Up @@ -2,55 +2,74 @@
use Cassandane::Tiny;

sub test_email_import_received_at
:min_version_3_1 :needs_component_sieve :needs_component_jmap
:needs_component_jmap
{
my ($self) = @_;
my $jmap = $self->{jmap};
my ($maj, $min) = Cassandane::Instance->get_version();

my @testCases = ({
creationId => 'clientSet1',
dateHeader => "Date: Sat, 1 Jan 2022 01:00:00 +1100\r\n",
receivedHeader => undef,
desc => 'receivedAt set by client',
headers => "Date: Sat, 1 Jan 2022 01:00:00 +1100\r\n",
receivedAt => '2022-02-01T12:00:00Z',
wantSentAt => '2022-01-01T01:00:00+11:00',
wantReceivedAt => '2022-02-01T12:00:00Z',
}, {
creationId => 'clientSet2',
dateHeader => "Date: Sat, 1 Jan 2022 01:00:00 +1100\r\n",
receivedHeader => "Received: from foo ([192.168.0.1]) by bar (Baz); Mon, 15 Aug 2022 07:49:01 -0400\r\n",
desc => 'receivedAt set by client',
headers => "Date: Sat, 1 Jan 2022 01:00:00 +1100\r\n" .
"Received: from foo ([192.168.0.1]) by bar (Baz); Mon, 15 Aug 2022 07:49:01 -0400\r\n",
receivedAt => '2022-02-01T12:00:00Z',
wantSentAt => '2022-01-01T01:00:00+11:00',
wantReceivedAt => '2022-02-01T12:00:00Z',
}, {
desc => 'receivedAt from Received header',
creationId => 'receivedAtFromReceivedHeader',
dateHeader => "Date: Sat, 1 Jan 2022 01:00:00 +1100\r\n",
receivedHeader => "Received: from foo ([192.168.0.1]) by bar (Baz); Mon, 15 Aug 2022 07:49:01 -0400\r\n",
receivedAt => undef,
headers => "Date: Sat, 1 Jan 2022 01:00:00 +1100\r\n" .
"Received: from foo ([192.168.0.1]) by bar (Baz); Mon, 15 Aug 2022 07:49:01 -0400\r\n",
wantSentAt => '2022-01-01T01:00:00+11:00',
wantReceivedAt => '2022-08-15T11:49:01Z',
skipVersionBefore => qw(3,7),
}, {
creationId => 'receivedAtFromFirstReceivedHeader',
dateHeader => "Date: Sat, 1 Jan 2022 01:00:00 +1100\r\n",
receivedHeader => "Received: from rcv1 ([192.168.0.1]) by bar (Baz); Mon, 15 Aug 2022 07:49:01 -0400\r\n" .
"Received: from rcv2 ([192.168.0.2]) by tux (Qux); Mon, 13 Aug 2022 12:01:10 -0200\r\n" .
"Received: from rcv3 ([192.168.0.3]) by baz (Hkl); Mon, 16 Aug 2022 13:01:10 -0200\r\n",
receivedAt => undef,
desc => 'receivedAt from first Received header',
headers => "Date: Sat, 1 Jan 2022 01:00:00 +1100\r\n" .
"Received: from rcv1 ([192.168.0.1]) by bar (Baz); Mon, 15 Aug 2022 07:49:01 -0400\r\n" .
"Received: from rcv2 ([192.168.0.2]) by tux (Qux); Sat, 13 Aug 2022 12:01:10 -0200\r\n" .
"Received: from rcv3 ([192.168.0.3]) by baz (Hkl); Tue, 16 Aug 2022 13:01:10 -0200\r\n",
wantSentAt => '2022-01-01T01:00:00+11:00',
wantReceivedAt => '2022-08-15T11:49:01Z',
skipVersionBefore => qw(3,7),
}, {
creationId => 'receivedAtFromDateHeader',
dateHeader => "Date: Sat, 1 Jan 2022 01:00:00 +1100\r\n",
receivedHeader => undef,
receivedAt => undef,
desc => 'receivedAt from Date header',
headers => "Date: Sat, 1 Jan 2022 01:00:00 +1100\r\n",
wantSentAt => '2022-01-01T01:00:00+11:00',
wantReceivedAt => '2021-12-31T14:00:00Z',
skipVersionBefore => qw(3,7),
}, {
creationId => 'serverSetNoHeader',
receivedHeader => undef,
receivedAt => undef,
desc => 'not set',
wantSentAt => undef,
wantReceivedAt => undef,
}, {
desc => 'receivedAt from first valid Received header',
headers => "Date: Sat, 1 Jan 2022 01:00:00 +1100\r\n" .
"Received: from rcv1 ([192.168.0.1]) by bar (Baz); invalid datetime\r\n" .
"Received: from rcv2 ([192.168.0.2]) by tux (Qux); Sat, 13 Aug 2022 12:01:10 -0200\r\n" .
"Received: from rcv3 ([192.168.0.3]) by baz (Hkl); Sat, 13 Aug 2022 12:00:45 -0200\r\n",
wantSentAt => '2022-01-01T01:00:00+11:00',
wantReceivedAt => '2022-08-13T14:01:10Z',
skipVersionBefore => qw(3,9),
}, {
desc => 'receivedAt from X-DeliveredInternalDate header',
headers => "Date: Sat, 1 Jan 2022 01:00:00 +1100\r\n" .
"Received: from rcv1 ([192.168.0.2]) by tux (Qux); Sat, 13 Aug 2022 12:01:10 -0200\r\n" .
"Received: from rcv2 ([192.168.0.3]) by baz (Hkl); Sat, 13 Aug 2022 12:00:45 -0200\r\n" .
"X-DeliveredInternalDate: Mon, 15 Aug 2022 13:00:45 -0200\r\n",
,
wantSentAt => '2022-01-01T01:00:00+11:00',
wantReceivedAt => '2022-08-15T15:00:45Z',
skipVersionBefore => qw(3,9),
});

foreach my $tc (@testCases) {
while (my ($i, $tc) = each @testCases) {

my ($needMaj, $needMin) = $tc->{skipVersionBefore} || qw(0,0);
if ($maj < $needMaj || ($maj == $needMaj && $min < $needMin)) {
Expand All @@ -61,8 +80,7 @@ sub test_email_import_received_at

xlog $self, "Running test $tc->{creationId}";

my $mime = $tc->{receivedHeader} || '';
$mime .= $tc->{dateHeader} || '';
my $mime = $tc->{headers} || '';
$mime .= <<'EOF';
From: sender@local
To: receiver@local
Expand All @@ -73,17 +91,19 @@ Content-Transfer-Encoding: quoted-printable
EOF
$mime =~ s/\r?\n/\r\n/gs;
$mime .= "\r\n";
$mime .= $tc->{creationId};
$mime .= $tc->{desc} || 'foo';

xlog $self, "Upload blob";
my $blobId = ($jmap->Upload($mime, 'message/rfc822'))->{blobId};
$self->assert_not_null($blobId);

xlog $tc->{creationId}, "Import mail";
my $creationId = "email" . ($i + 1);

xlog $self, "Import $creationId" . (": $tc->{desc}" || '');
my $res = $jmap->CallMethods([
['Email/import', {
emails => {
$tc->{creationId} => {
$creationId => {
blobId => $blobId,
mailboxIds => {
'$inbox' => JSON::true
Expand All @@ -94,17 +114,17 @@ EOF
}, 'R1'],
['Email/get', {
ids => [
'#' . $tc->{creationId},
"#$creationId",
],
properties => ['receivedAt', 'sentAt'],
}, 'R2']
]);

$self->assert_not_null($res->[0][1]{created}{$tc->{creationId}});
$self->assert_not_null($res->[0][1]{created}{$creationId});

xlog $self, "Assert sentAt";
if ($tc->{dateHeader}) {
$self->assert_str_equals('2022-01-01T01:00:00+11:00',
if ($tc->{wantSentAt}) {
$self->assert_str_equals($tc->{wantSentAt},
$res->[1][1]{list}[0]{sentAt});
} else {
$self->assert_null($res->[1][1]{list}[0]{sentAt});
Expand Down
71 changes: 55 additions & 16 deletions imap/jmap_mail.c
View file
Expand Up @@ -13236,6 +13236,57 @@ static int msgimport_checkacl_cb(const mbentry_t *mbentry, void *xrock)
return 0;
}

static time_t _email_import_parse_received_at(const char *blob, size_t blob_len)
{
message_t *msg = message_new_from_data(blob, blob_len);
struct buf buf = BUF_INITIALIZER;
time_t received_at = 0;
enum message_format format = MESSAGE_DECODED | MESSAGE_TRIM;

if (!msg)
goto done;

if (!message_get_field(msg, "X-Deliveredinternaldate", format, &buf)) {
buf_appendcstr(&buf, "\r\n");
const char *hdr = buf_cstring(&buf);
char *val = NULL;
message_parse_string(hdr, &val);
if (time_from_rfc822(val, &received_at) < 0)
received_at = 0;
xzfree(val);
}

if (!received_at && !message_get_field(msg, "Received", format, &buf)) {
buf_appendcstr(&buf, "\r\n");
const char *hdr = buf_cstring(&buf);
char *val = NULL;
do {
message_parse_received_date(hdr, &val);
if (time_from_rfc822(val, &received_at) < 0)
received_at = 0;
hdr = strchr(hdr, '\n');
if (hdr)
hdr = strchr(hdr + 1, ':');
xzfree(val);
} while (!received_at && hdr++);
}

if (!received_at && !message_get_field(msg, "Date", format, &buf)) {
buf_appendcstr(&buf, "\r\n");
const char *hdr = buf_cstring(&buf);
char *val = NULL;
message_parse_string(hdr, &val);
if (time_from_rfc822(val, &received_at) < 0)
received_at = 0;
xzfree(val);
}

done:
message_unref(&msg);
buf_free(&buf);
return received_at;
}

static void _email_import(jmap_req_t *req,
json_t *jemail_import,
json_t **new_email,
Expand Down Expand Up @@ -13356,26 +13407,14 @@ static void _email_import(jmap_req_t *req,

/* set receivedAt property */
time_t internaldate = 0;
const char *received_at = json_string_value(json_object_get(jemail_import, "receivedAt"));
const char *received_at =
json_string_value(json_object_get(jemail_import, "receivedAt"));
if (received_at) {
time_from_iso8601(received_at, &internaldate);
}
else {
/* check for Received and Date Header */
struct body *mybody = xzmalloc(sizeof(struct body));
r = message_parse_mapped(buf_base(&content), buf_len(&content), mybody, NULL);
if (!r) {
const char *date = mybody->received_date ?
mybody->received_date : mybody->date;
if (date) {
time_t t = 0;
if (time_from_rfc822(date, &t) > 0) {
internaldate = t;
}
}
}
message_free_body(mybody);
free(mybody);
internaldate = _email_import_parse_received_at(buf_base(&content),
buf_len(&content));
}
if (!internaldate)
internaldate = time(NULL);
Expand Down
3 changes: 1 addition & 2 deletions imap/message.c
View file
Expand Up @@ -128,7 +128,6 @@ static void message_parse_params(const char *hdr, struct param **paramp);
static void message_fold_params(struct param **paramp);
static void message_parse_language(const char *hdr, struct param **paramp);
static void message_parse_rfc822space(const char **s);
static void message_parse_received_date(const char *hdr, char **hdrp);

static void message_parse_multipart(struct msg *msg,
struct body *body,
Expand Down Expand Up @@ -1987,7 +1986,7 @@ static void message_parse_content(struct msg *msg, struct body *body,
body_add_content_guid(msg->base + s_offset, body);
}

static void message_parse_received_date(const char *hdr, char **hdrp)
EXPORTED void message_parse_received_date(const char *hdr, char **hdrp)
{
char *curp, *hdrbuf = 0;

Expand Down
2 changes: 2 additions & 0 deletions imap/message.h
View file
Expand Up @@ -201,6 +201,8 @@ extern void message_parse_disposition(const char *hdr, char **hdpr, struct param

extern void message_parse_charset_params(const struct param *params, charset_t *c_ptr);

extern void message_parse_received_date(const char *hdr, char **hdrp);

/* NOTE - scribbles on its input */
extern void message_parse_env_address(char *str, struct address *addr);

Expand Down

0 comments on commit 19961c5

Please sign in to comment.