read(write(record)) != record

[PromyLOPh]

Records read back from a file just written should be equal to the Python object written. This is something I discovered while writing tests for an application using warcio. Test case:

import pytest

from io import BytesIO
from tempfile import NamedTemporaryFile

from warcio.archiveiterator import ArchiveIterator
from warcio.warcwriter import WARCWriter
from warcio.statusandheaders import StatusAndHeaders

def test_identity ():
    """ read(write(record)) should yield record """
    with NamedTemporaryFile () as fd:
        payload = b'foobar'
        writer = WARCWriter (fd, gzip=True)
        httpHeaders = StatusAndHeaders('GET / HTTP/1.1', {}, is_http_request=True)
        warcHeaders = {'Foo': 'Bar'}
        record = writer.create_warc_record ('http://example.com/', 'request',
                payload=BytesIO(payload),
                warc_headers_dict=warcHeaders, http_headers=httpHeaders)
        writer.write_record (record)

        fd.seek (0)
        rut = next (ArchiveIterator (fd))
        golden = record
        assert rut.rec_type == golden.rec_type
        assert rut.rec_headers == golden.rec_headers
        assert rut.content_type == golden.content_type
        assert rut.length == golden.length
        assert rut.http_headers == golden.http_headers
        assert rut.raw_stream.read() == payload

results in the following assertion failure:

E           AssertionError: assert StatusAndHead...ngth', '24')]) == StatusAndHeade...ngth', '24')])
E             Full diff:
E             - StatusAndHeaders(protocol = 'WARC/1.0', statusline = '', headers = [('Foo', 'Bar'), ('WARC-Type', 'request'), ('WARC-Record-ID', '<urn:uuid:2eb39603-c759-4865-9e5b-2a3cd9c81c92>'), ('WARC-Target-URI', 'http://example.com/'), ('WARC-Date', '2018-12-04T15:27:45Z'), ('WARC-Payload-Digest', 'sha1:RBB5P6JECYQR32PLXFR76THCQESZGKDY'), ('WARC-Block-Digest', 'sha1:HVUJ5SESVATOLVXZZTFORJY44V5BW7YB'), ('Content-Type', 'application/http; msgtype=request'), ('Content-Length', '24')])
E             ?                                         ^^^^^^^^^^   --------------
E             + StatusAndHeaders(protocol = '', statusline = 'WARC/1.0', headers = [('Foo', 'Bar'), ('WARC-Type', 'request'), ('WARC-Record-ID', '<urn:uuid:2eb39603-c759-4865-9e5b-2a3cd9c81c92>'), ('WARC-Target-URI', 'http://example.com/'), ('WARC-Date', '2018-12-04T15:27:45Z'), ('WARC-Payload-Digest', 'sha1:RBB5P6JECYQR32PLXFR76THCQESZGKDY'), ('WARC-Block-Digest', 'sha1:HVUJ5SESVATOLVXZZTFORJY44V5BW7YB'), ('Content-Type', 'application/http; msgtype=request'), ('Content-Length', '24')])
E             ?                            +++++++++++++++++             ^^^^^^^

[ikreymer]

This should now be a fixed, I added a unit test based on your example above:
https://github.com/webrecorder/warcio/blob/develop/test/test_writer.py#L826

Released in 1.7.2