Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SIDD byte provider #219

Merged
merged 37 commits into from
Jan 24, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
0499775
Switching to size_t to reflect sizes
asylvest Sep 24, 2017
b627fe5
Moved implementation of getActualDim() into nitf::ImageSubheader
asylvest Sep 24, 2017
b1a0b6f
Getting rid of compiler warnings by using correct types
asylvest Sep 24, 2017
37fd9ec
Moved buffer management logic into nitf module. Moved core byte mani…
asylvest Sep 24, 2017
561231e
Local changes that'll flow into NITRO eventually
asylvest Sep 24, 2017
a58b4a7
Adding initial cut at ImageBlocker - still lots more to do
asylvest Sep 25, 2017
23d81e8
Adding comment
asylvest Sep 25, 2017
06601bc
getNumBytesRequired() implemented
asylvest Oct 14, 2017
22f58bf
Added first blockImpl()
asylvest Oct 14, 2017
26bade2
Another big chunk of implementation done
asylvest Dec 17, 2017
5483b1b
Starting to add some unittests and fixing bugs as we go
asylvest Dec 17, 2017
4571488
More tests and fixes
asylvest Dec 17, 2017
630eef7
Fixed line endings
asylvest Dec 18, 2017
52667da
Added overloading for just one segment
asylvest Dec 18, 2017
c518d58
Using other constructor
asylvest Dec 18, 2017
73323a4
Unittests for multiple image segments case
asylvest Dec 24, 2017
1ca5144
Got partial image test working
asylvest Dec 31, 2017
c66ba57
Finishing up unittests
asylvest Dec 31, 2017
72b8547
Merge branch 'master' into sidd_byte_provider
asylvest Jan 1, 2018
fe07f43
Adding SIDDByteProvider
asylvest Jan 1, 2018
a5dd88f
Using new io module function for this
asylvest Jan 1, 2018
970abdb
Fixing banner
asylvest Jan 1, 2018
8908692
Adding unittests
asylvest Jan 1, 2018
c550521
Partially adding in support for blocking
asylvest Jan 2, 2018
cce5011
Adding better block indexing
asylvest Jan 6, 2018
01c385a
Adding checks to ensure hits block boundaries
asylvest Jan 6, 2018
66ebc83
Starting to add block-specific test
asylvest Jan 7, 2018
0aa1c67
Merge branch 'master' into sidd_byte_provider
asylvest Jan 7, 2018
63e1c90
Unit test is close
asylvest Jan 13, 2018
d24df57
Flipped tests around
asylvest Jan 13, 2018
e57408d
Fixed range check
asylvest Jan 14, 2018
43eb951
Fixing one more corner case
asylvest Jan 21, 2018
3a842ba
Added to regression test
asylvest Jan 21, 2018
a5845aa
Updating comments
asylvest Jan 21, 2018
1e047b2
Fixed spacing
asylvest Jan 22, 2018
42cd29e
Clarified variable name
asylvest Jan 22, 2018
74e8fb1
Merge branch 'master' into sidd_byte_provider
asylvest Jan 24, 2018
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions six/modules/c++/six.sicd/include/six/sicd/RadarCollection.h
Original file line number Diff line number Diff line change
Expand Up @@ -251,15 +251,15 @@ struct Segment
Segment* clone() const;

//! The number of lines in the segment
int getNumLines() const
size_t getNumLines() const
{
// Rotating can make the start/end in reverse order,
// so need the absolute value
return std::abs(endLine - startLine) + 1;
}

//! The number of samples in the segment
int getNumSamples() const
size_t getNumSamples() const
{
return std::abs(endSample - startSample) + 1;
}
Expand Down
222 changes: 2 additions & 220 deletions six/modules/c++/six.sicd/include/six/sicd/SICDByteProvider.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,7 @@
#ifndef __SIX_SICD_BYTE_PROVIDER_H__
#define __SIX_SICD_BYTE_PROVIDER_H__

#include <vector>

#include <six/NITFSegmentInfo.h>
#include <six/NITFWriteControl.h>
#include <six/ByteProvider.h>
#include <six/sicd/ComplexData.h>

namespace six
Expand All @@ -48,143 +45,9 @@ namespace sicd
* rather than one contiguous block of memory in order to not perform any
* copies.
*/
class SICDByteProvider
class SICDByteProvider : public six::ByteProvider
{
public:
/*!
* \class NITFBuffer
* \brief Represents a pointer to raw NITF bytes and its length
*/
struct NITFBuffer
{
/*!
* Initializes to an empty buffer
*/
NITFBuffer();

/*!
* Initializes to the specified pointer and size. No copy is made and
* this object does not take ownership.
*
* \param data The raw bytes of data
* \param numBytes The number of bytes of contiguous data this
* represents
*/
NITFBuffer(const void* data, size_t numBytes);

const void* mData;
size_t mNumBytes;
};

/*!
* \class NITFBufferList
* \brief Represents a sequence of buffers which appear in contiguous order
* in the NITF (the underlying pointers are not contiguous)
*/
struct NITFBufferList
{
//! The buffers
std::vector<NITFBuffer> mBuffers;

/*!
* \return The total number of bytes across all the buffers
*/
size_t getTotalNumBytes() const;

/*!
* \return Whether or not the buffer list is empty
*/
bool empty() const
{
return mBuffers.empty();
}

/*!
* Clear the buffers
*/
void clear()
{
mBuffers.clear();
}

/*!
* Push data onto the buffer list
*
* \param data The raw bytes
* \param numBytes The number of bytes of data
*/
void pushBack(const void* data, size_t numBytes)
{
mBuffers.push_back(NITFBuffer(data, numBytes));
}

/*!
* Push data onto the buffer list
*
* \tparam DataT The type of data
*
* \param data The raw bytes
*/
template <typename DataT>
void pushBack(const std::vector<DataT>& data)
{
pushBack(data.empty() ? NULL : &data[0],
data.size() * sizeof(DataT));
}

/*!
* Get the number of blocks of data of size 'blockSize'. In cases
* where the block size is not an even multiple of the total number of
* bytes, the last block will be larger than the block size (rather
* than there being one more block which is smaller than the block
* size). This is intentional in order to make this easily usable with
* Amazon's S3 storage with multipart uploads where there is a minimum
* part size (there may be multiple machines, all with their portion
* of the SICD, performing a multipart upload of their parts, and
* only the last overall part of the object can be less than the
* minimum part size).
*
* \param blockSize The desired block size
*
* \return The associated number of blocks
*/
size_t getNumBlocks(size_t blockSize) const;

/*!
* Get the number of bytes in the specified block. All blocks will be
* the same size except for the last block (see getNumBlocks() for
* details).
*
* \param blockSize The desired block size
* \param blockIdx The 0-based block index
*
* \return The number of bytes in this block
*/
size_t getNumBytesInBlock(size_t blockSize, size_t blockIdx) const;

/*!
* Returns a pointer to contiguous memory associated with the desired
* block. If this block lies entirely within a NITFBuffer, no copy
* is performed. Otherwise, the scratch buffer is resized, the bytes
* are copied into this, and a pointer to the scratch buffer is
* returned.
*
* \param blockSize The desired block size. See getNumBlocks() for a
* description on the behavior of the last block.
* \param blockIdx The 0-based block index
* \param[out] scratch Scratch buffer. This will be resized and used
* if the underlying memory for this block is not contiguous (i.e. it
* spans NITFBuffers).
* \param[out] numBytes The number of bytes in this block
*
* \return A pointer to contiguous memory associated with this block
*/
const void* getBlock(size_t blockSize,
size_t blockIdx,
std::vector<sys::byte>& scratch,
size_t& numBytes) const;
};

/*!
* Constructor
*
Expand All @@ -207,87 +70,6 @@ class SICDByteProvider
*/
SICDByteProvider(const NITFWriteControl& writer,
const std::vector<std::string>& schemaPaths);

/*!
* \return The total number of bytes in the NITF
*/
nitf::Off getFileNumBytes() const
{
return mFileNumBytes;
}

/*!
* Given a range of rows from [startRow, startRow + numRows), provide the
* number of bytes that will appear in the NITF on disk (including NITF
* file header, image subheader(s), and DES subheader and data). Calling
* this method repeatedly, eventually providing the entire range of the
* image, will produce the total number of bytes in the full NITF.
*
* \param startRow The global start row in pixels as to where these pixels
* are in the image. If this is a multi-segment NITF, this is still simply
* the global pixel location.
* \param numRows The number of rows
*
* \return The associated number of bytes in the NITF
*/
nitf::Off getNumBytes(size_t startRow, size_t numRows) const;

/*!
* The caller provides an AOI of the pixel data. This method provides back
* a list of contiguous buffers corresponding to the raw NITF bytes for
* this portion of the file. If this AOI is in the middle of an image
* segment, this will be simply a buffer list of length 1 consisting of the
* input pointer (no copy occurs). Otherwise, pointers to various headers
* (file header, image subheader(s), DES subheader and data) will
* be in the buffer list before, in the middle, and/or after the image
* data. If this method is called multiple times with AOIs that
* eventually consist of the entire image, and the raw bytes are written
* out to disk in order with respect to the start pixel rows this method is
* called with (or out of order but seeking to the provided file offset
* each time), and in the order contained in the buffer list, it will form
* a valid NITF.
*
* \note This method does not perform byte swapping on the pixel data for
* efficiency reasons, but NITFs are written out in big endian order. This
* means that on a little endian system, you must byte swap the pixel data
* prior to calling this method.
*
* \param imageData The image data pixels to write. The underlying type
* will be complex short or complex float based on the complex data sent
* in during initialize. Must be in big endian order.
* \param startRow The global start row in pixels as to where these pixels
* are in the image. If this is a multi-segment NITF, this is still simply
* the global pixel location.
* \param numRows The number of rows in the provided 'imageData'
* \param[out] fileOffset The offset in bytes in the NITF where these
* buffers should be written
* \param[out] buffers One or more pointers to raw bytes of data. These
* should be written out in the order they are provided in the buffer list.
* The pointers point to the provided pixel data and, if required, one or
* more NITF headers. No copies occur, so these buffers are only valid for
* the lifetime of this provider object (since this object owns the raw
* bytes for the NITF headers) and the lifetime of the passed-in image
* data.
*/
void getBytes(const void* imageData,
size_t startRow,
size_t numRows,
nitf::Off& fileOffset,
NITFBufferList& buffers) const;

private:
static size_t getNumBytesPerRow(const NITFWriteControl& writer);

private:
const size_t mNumBytesPerRow;

std::vector<sys::byte> mFileHeader;
std::vector<std::vector<sys::byte> > mImageSubheaders;
std::vector<sys::byte> mDesSubheaderAndData;
std::vector<nitf::Off> mImageSubheaderFileOffsets;
std::vector<NITFSegmentInfo> mImageSegmentInfo;
nitf::Off mDesSubheaderFileOffset;
nitf::Off mFileNumBytes;
};
}
}
Expand Down
20 changes: 10 additions & 10 deletions six/modules/c++/six.sicd/source/AreaPlaneUtility.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,11 @@ std::vector<RowColDouble > AreaPlaneUtility::computeCornersPix(
// Use the grid corners
cornersPix.push_back(RowColDouble(0.0, 0.0));
cornersPix.push_back(RowColDouble(
0.0, data.getNumCols() - 1));
0.0, data.getNumCols() - 1.0));
cornersPix.push_back(RowColDouble(
data.getNumRows() - 1, data.getNumCols() - 1));
data.getNumRows() - 1.0, data.getNumCols() - 1.0));
cornersPix.push_back(RowColDouble(
data.getNumRows() - 1, 0.0));
data.getNumRows() - 1.0, 0.0));
}

return cornersPix;
Expand Down Expand Up @@ -221,11 +221,11 @@ void AreaPlaneUtility::setAreaPlane(ComplexData& data,
}

std::vector<RowColDouble> imageCorners(4);
imageCorners[0] = RowColDouble(0, 0);
imageCorners[1] = RowColDouble(0, data.getNumCols() - 1);
imageCorners[2] = RowColDouble(data.getNumRows() - 1,
data.getNumCols() - 1);
imageCorners[3] = RowColDouble(data.getNumRows() - 1, 0);
imageCorners[0] = RowColDouble(0.0, 0.0);
imageCorners[1] = RowColDouble(0.0, data.getNumCols() - 1.0);
imageCorners[2] = RowColDouble(data.getNumRows() - 1.0,
data.getNumCols() - 1.0);
imageCorners[3] = RowColDouble(data.getNumRows() - 1.0, 0.0);
LatLonAltCorners& acpCorners =
data.radarCollection->area->acpCorners;

Expand Down Expand Up @@ -300,9 +300,9 @@ void AreaPlaneUtility::deriveAreaPlane(const ComplexData& data,

// End values are inclusive
areaPlane.segmentList[0]->endLine =
areaPlane.xDirection->elements - 1;
static_cast<int>(areaPlane.xDirection->elements - 1);
areaPlane.segmentList[0]->endSample =
areaPlane.yDirection->elements - 1;
static_cast<int>(areaPlane.yDirection->elements - 1);

areaPlane.segmentList[0]->identifier = "AA";
}
Expand Down
Loading