Separate CSVF conversion & compress operations. Make private. Stop co…

…mpressing when not necessary. Changes forced by underlying libbuffer changes.

csvfplay.c

@@ -35,7 +35,7 @@ static void dumpSimple(const unsigned char *input, unsigned int length, char *p)
 
 // Play the uncompressed CSVF stream into the JTAG port.
 //
-int csvfPlay(const uint8 *csvfData, struct NeroHandle *nero, const char **error) {
+int csvfPlay(const uint8 *csvfData, bool isCompressed, struct NeroHandle *nero, const char **error) {
 	int returnCode;
 	NeroStatus nStatus;
 	uint8 thisByte;
@@ -54,7 +54,7 @@ int csvfPlay(const uint8 *csvfData, struct NeroHandle *nero, const char **error)
 	nStatus = neroClockFSM(nero, 0x0000001F, 6, error);  // Reset TAP, goto Run-Test/Idle
 	CHECK_STATUS(nStatus, "csvfPlay()", nStatus);
 
-	if ( csvfInitReader(&cp, csvfData) ) {
+	if ( csvfInitReader(&cp, csvfData, isCompressed) ) {
 		// Header byte may be used for something later. For now, ensure it's zero.
 		errRender(error, "csvfPlay(): Bad CSVF header!");
 		FAIL(CPLAY_HEADER_ERR);

csvfplay.h

@@ -33,7 +33,7 @@ extern "C" {
 
 	// Play the uncompressed CSVF stream into the JTAG port.
 	int csvfPlay(
-		const uint8 *csvfData, struct NeroHandle *nero, const char **error
+		const uint8 *csvfData, bool isCompressed, struct NeroHandle *nero, const char **error
 	) WARN_UNUSED_RESULT;
 
 #ifdef __cplusplus

csvfreader.c

@@ -30,18 +30,28 @@ static uint32 readLength(struct Context *cp);
 
 // Init the reader, return the header byte
 //
-uint8 csvfInitReader(struct Context *cp, const uint8 *csvfData) {
+uint8 csvfInitReader(struct Context *cp, const uint8 *csvfData, bool isCompressed) {
 	uint8 thisByte;
 	cp->data = csvfData;
-	thisByte = getRawByte(cp);
-	cp->count = readLength(cp);
+	if ( isCompressed ) {
+		cp->isCompressed = true;
+		thisByte = getRawByte(cp);
+		cp->count = readLength(cp);
+	} else {
+		cp->isCompressed = false;
+		thisByte = 0x00;
+		cp->count = 0;
+	}
 	cp->isReadingChunk = true;
 	return thisByte;
 }
 
 // Get the next byte from the uncompressed stream. Uses m_count & m_isReadingChunk to keep state.
 //
 uint8 csvfGetByte(struct Context *cp) {
+	if ( !cp->isCompressed ) {
+		return getRawByte(cp);
+	}
 	if ( cp->isReadingChunk ) {
 		// We're in the middle of reading a chunk.
 		if ( cp->count ) {

csvfreader.h

@@ -27,9 +27,10 @@ extern "C" {
 		const uint8 *data;
 		uint32 count;
 		bool isReadingChunk;
+		bool isCompressed;
 	};
 
-	uint8 csvfInitReader(struct Context *cp, const uint8 *data);
+	uint8 csvfInitReader(struct Context *cp, const uint8 *data, bool isCompressed);
 	uint8 csvfGetByte(struct Context *cp);
 	uint16 csvfGetWord(struct Context *cp);
 	uint32 csvfGetLong(struct Context *cp);

examples/python/flashFirmware.sh

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 #!/bin/bash
+export PYTHON_VERSION=2
 echo
 echo "WARNING: this will erase any firmware currently in your FX2FPGA EEPROM!!!"
 echo
@@ -25,14 +26,14 @@ read -n 1 -s
 echo
 echo "Writing firmware to EEPROM now..."
 sudo LD_LIBRARY_PATH=../../linux.x86_64/rel python <<EOF
-from fpgalink import *
+from fpgalink${PYTHON_VERSION} import *
 flLoadStandardFirmware("04B4:8613", "04B4:8613")
 flAwaitDevice("04B4:8613", 600)
 handle = flOpen("04B4:8613")
 flAppendWriteRegisterCommand(handle, 0x00, 0x10)
 flAppendWriteRegisterCommand(handle, 0x00, 0x10)
 flAppendWriteRegisterCommand(handle, 0x00, 0x10)
-flFlashStandardFirmware(handle, "04B4:8613", 512, "../../gen_xsvf/s3board.xsvf")
+flFlashStandardFirmware(handle, "04B4:8613", 512, "../../gen_csvf/s3board.csvf")
 flClose(handle)
 quit()
 EOF
@@ -45,7 +46,7 @@ echo "6) Press any key"
 read -n 1 -s
 
 sudo LD_LIBRARY_PATH=../../linux.x86_64/rel python <<EOF
-from fpgalink import *
+from fpgalink${PYTHON_VERSION} import *
 handle = flOpen("04B4:8613")
 flWriteRegister(handle, 1000, 0x00, 0x10)
 flWriteRegister(handle, 1000, 0x00, 0x10)

firmware/jtag.c

@@ -754,12 +754,8 @@ uint8 jtagCsvfPlay(void) {
 			break;
 		}
 
-		case XSDRB:
+		case XSDR:
 			jtagClockFSM(0x00000001, 3);
-			shiftOutCsvf(xsdrSize, false);
-			break;
-
-		case XSDRE:
 			shiftOutCsvf(xsdrSize, true);
 			jtagClockFSM(0x00000001, 2);
 			if ( xruntest ) {

fx2.c

@@ -150,7 +150,7 @@ DLLEXPORT(FLStatus) flFlashStandardFirmware(
 	}
 
 	//bStatus = bufWriteBinaryFile(&i2cBuf, "out.bin", 0UL, i2cBuf.length, error);
-	//CHECK_STATUS(bStatus, "flFlashStandardFirmware()");
+	//CHECK_STATUS(bStatus, "flFlashStandardFirmware()", FL_ALLOC_ERR);
 
 	fxStatus = fx2WriteEEPROM(handle->device, i2cBuf.data, i2cBuf.length, error);
 	CHECK_STATUS(fxStatus, "flFlashStandardFirmware()", FL_FX2_ERR);
@@ -329,12 +329,14 @@ static FLStatus appendCsvfFromXsvf(struct Buffer *dest, const char *xsvfFile, co
 	if ( strcmp(".xsvf", ext) == 0 ) {
 		bStatus = bufInitialise(&csvfBuf, 0x20000, 0, error);
 		CHECK_STATUS(bStatus, "appendCsvfFromXsvf()", FL_ALLOC_ERR);
-		fStatus = flLoadXsvfAndConvertToCsvf(xsvfFile, &csvfBuf, &maxBufSize, NULL, error);
+		fStatus = flLoadXsvfAndConvertToCsvf(xsvfFile, &csvfBuf, &maxBufSize, error);
 		CHECK_STATUS(fStatus, "appendCsvfFromXsvf()", fStatus);
 		if ( maxBufSize > CSVF_BUF_SIZE ) {
 			errRender(error, "appendCsvfFromXsvf(): This XSVF file requires CSVF_BUF_SIZE=%d", maxBufSize);
 			FAIL(FL_JTAG_ERR);
 		}
+		fStatus = flCompressCsvf(&csvfBuf, error);
+		CHECK_STATUS(fStatus, "appendCsvfFromXsvf()", fStatus);
 		bStatus = bufAppendBlock(dest, csvfBuf.data, csvfBuf.length, error);
 		CHECK_STATUS(bStatus, "appendCsvfFromXsvf()", FL_ALLOC_ERR);
 	} else if ( strcmp(".csvf", ext) == 0 ) {

jtag.c

@@ -32,6 +32,7 @@ DLLEXPORT(FLStatus) flPlayXSVF(struct FLContext *handle, const char *xsvfFile, c
 	NeroStatus nStatus;
 	int cStatus;
 	uint32 maxBufSize;
+	bool isCompressed;
 	const char *const ext = xsvfFile + strlen(xsvfFile) - 5;
 	if ( !handle->isNeroCapable ) {
 		errRender(error, "flPlayXSVF(): This device does not support NeroJTAG");
@@ -40,18 +41,20 @@ DLLEXPORT(FLStatus) flPlayXSVF(struct FLContext *handle, const char *xsvfFile, c
 	bStatus = bufInitialise(&csvfBuf, 0x20000, 0, error);
 	CHECK_STATUS(bStatus, "flPlayXSVF()", FL_ALLOC_ERR);
 	if ( strcmp(".xsvf", ext) == 0 ) {
-		fStatus = flLoadXsvfAndConvertToCsvf(xsvfFile, &csvfBuf, &maxBufSize, NULL, error);
+		fStatus = flLoadXsvfAndConvertToCsvf(xsvfFile, &csvfBuf, &maxBufSize, error);
 		CHECK_STATUS(fStatus, "flPlayXSVF()", fStatus);
+		isCompressed = false;
 	} else if ( strcmp(".csvf", ext) == 0 ) {
 		bStatus = bufAppendFromBinaryFile(&csvfBuf, xsvfFile, error);
 		CHECK_STATUS(bStatus, "flPlayXSVF()", FL_FILE_ERR);
+		isCompressed = true;
 	} else {
 		errRender(error, "flPlayXSVF(): Filename should have .xsvf or .csvf extension");
 		FAIL(FL_FILE_ERR);
 	}
 	nStatus = neroInitialise(handle->device, &nero, error);
 	CHECK_STATUS(nStatus, "flPlayXSVF()", FL_JTAG_ERR);
-	cStatus = csvfPlay(csvfBuf.data, &nero, error);
+	cStatus = csvfPlay(csvfBuf.data, isCompressed, &nero, error);
 	CHECK_STATUS(cStatus, "flPlayXSVF()", FL_JTAG_ERR);
 	returnCode = FL_SUCCESS;
 cleanup:

libfpgalink.h

@@ -634,48 +634,8 @@ extern "C" {
 	DLLEXPORT(FLStatus) flPortAccess(
 		struct FLContext *handle, uint16 portWrite, uint16 ddr, uint16 *portRead, const char **error
 	) WARN_UNUSED_RESULT;
-
-	/**
-	 * @brief Load an XSVF file and convert it to CSVF
-	 *
-	 * XSVF is a weird format. It represents everything (even long bit-sequences) in big-endian,
-	 * which is daft because to play a bit-sequence into the JTAG chain, you have to seek forward
-	 * to the end of the bit-sequence, then read backwards until you get to the beginning, playing
-	 * bytes into the JTAG chain as you go. This is fine on a powerful desktop computer with lots of
-	 * memory, but a resource-constrained microcontroller would have trouble.
-	 *
-	 * The CSVF format ("Compressed Serial Vector Format") swaps the byte order so bit sequences
-	 * can be played by reading forwards, and it replaces each XSDRB, XSDRC*, XSDRE command sequence
-	 * with one big XSDR command. It also does compression using run-length encoding.
-	 *
-	 * @param xsvfFile The XSVF filename.
-	 * @param csvfBuf A pointer to a \c Buffer to be populated with the CSVF data.
-	 * @param maxBufSize A pointer to a \c uint32 which will be set on exit to the number of bytes
-	 *            necessary for buffering in the playback logic. If this is greater than the
-	 *            \c CSVF_BUF_SIZE defined for the firmware, bad things will happen.
-	 * @param uncompressedBuf If set, will provide a view of the intermediate stage in the
-	 *            conversion, immediately prior to the RLE compression. Usually \c NULL.
-	 * @param error A pointer to a <code>char*</code> which will be set on exit to an allocated
-	 *            error message if something goes wrong. Responsibility for this allocated memory
-	 *            passes to the caller and must be freed with \c flFreeError(). If \c error is
-	 *            \c NULL, no allocation is done and no message is returned, but the return code
-	 *            will still be valid.
-	 * @returns
-	 *     - \c FL_SUCCESS if the command completed successfully.
-	 *     - \c FL_BUF_INIT_ERR If the CSVF buffer could not be allocated.
-	 *     - \c FL_BUF_APPEND_ERR If the CSVF buffer could not be grown.
-	 *     - \c FL_BUF_LOAD_ERR If the XSVF file could not be loaded.
-	 *     - \c FL_UNSUPPORTED_CMD_ERR If the XSVF file contains an unsupported command.
-	 *     - \c FL_UNSUPPORTED_DATA_ERR if the XSVF file contains an unsupported XENDIR or XENDDR.
-	 *     - \c FL_UNSUPPORTED_SIZE_ERR if the XSVF file requires more buffer space than is available.
-	 */
-	DLLEXPORT(FLStatus) flLoadXsvfAndConvertToCsvf(
-		const char *xsvfFile, struct Buffer *csvfBuf, uint32 *maxBufSize,
-		struct Buffer *uncompressedBuf, const char **error
-	) WARN_UNUSED_RESULT;
 	//@}
 
-
 #ifdef __cplusplus
 }
 #endif

private.h

@@ -68,6 +68,61 @@ extern "C" {
 	void flWriteWord(uint16 value, uint8 *p);
 	void flWriteLong(uint32 value, uint8 *p);
 
+	/**
+	 * @brief Load an XSVF file and convert it to CSVF
+	 *
+	 * XSVF is a weird format. It represents everything (even long bit-sequences) in big-endian,
+	 * which is daft because to play a bit-sequence into the JTAG chain, you have to seek forward
+	 * to the end of the bit-sequence, then read backwards until you get to the beginning, playing
+	 * bytes into the JTAG chain as you go. This is fine on a powerful desktop computer with lots of
+	 * memory, but a resource-constrained microcontroller would have trouble.
+	 *
+	 * The CSVF format ("Compressed Serial Vector Format") swaps the byte order so bit sequences
+	 * can be played by reading forwards, and it replaces each XSDRB, XSDRC*, XSDRE command sequence
+	 * with one big XSDR command. CSVF buffers should be compressed with /c flCompressCsvf() before
+	 * being written to persistent storage.
+	 *
+	 * @param xsvfFile The XSVF filename.
+	 * @param csvfBuf A pointer to a \c Buffer to be populated with the CSVF data.
+	 * @param maxBufSize A pointer to a \c uint32 which will be set on exit to the number of bytes
+	 *            necessary for buffering in the playback logic. If this is greater than the
+	 *            \c CSVF_BUF_SIZE defined for the firmware, bad things will happen.
+	 * @param error A pointer to a <code>char*</code> which will be set on exit to an allocated
+	 *            error message if something goes wrong. Responsibility for this allocated memory
+	 *            passes to the caller and must be freed with \c flFreeError(). If \c error is
+	 *            \c NULL, no allocation is done and no message is returned, but the return code
+	 *            will still be valid.
+	 * @returns
+	 *     - \c FL_SUCCESS if the command completed successfully.
+	 *     - \c FL_BUF_INIT_ERR If the CSVF buffer could not be allocated.
+	 *     - \c FL_BUF_APPEND_ERR If the CSVF buffer could not be grown.
+	 *     - \c FL_BUF_LOAD_ERR If the XSVF file could not be loaded.
+	 *     - \c FL_UNSUPPORTED_CMD_ERR If the XSVF file contains an unsupported command.
+	 *     - \c FL_UNSUPPORTED_DATA_ERR if the XSVF file contains an unsupported XENDIR or XENDDR.
+	 *     - \c FL_UNSUPPORTED_SIZE_ERR if the XSVF file requires more buffer space than is available.
+	 */
+	DLLEXPORT(FLStatus) flLoadXsvfAndConvertToCsvf(
+		const char *xsvfFile, struct Buffer *csvfBuf, uint32 *maxBufSize, const char **error
+	) WARN_UNUSED_RESULT;
+
+	/**
+	 * @brief Perform run-length encoding on a CSVF buffer.
+	 *
+	 * @param csvfBuf A pointer to a \c Buffer containing CSVF data to be RLE-compressed.
+	 * @param error A pointer to a <code>char*</code> which will be set on exit to an allocated
+	 *            error message if something goes wrong. Responsibility for this allocated memory
+	 *            passes to the caller and must be freed with \c flFreeError(). If \c error is
+	 *            \c NULL, no allocation is done and no message is returned, but the return code
+	 *            will still be valid.
+	 * @returns
+	 *     - \c FL_SUCCESS if the command completed successfully.
+	 *     - \c FL_BUF_INIT_ERR If the compress buffer could not be allocated.
+	 *     - \c FL_BUF_APPEND_ERR If the compress buffer could not be grown.
+	 */
+	DLLEXPORT(FLStatus) flCompressCsvf(
+		struct Buffer *csvfBuf, const char **error
+	) WARN_UNUSED_RESULT;
+
 #ifdef __cplusplus
 }
 #endif

tests-integration/testCore.cpp

@@ -190,13 +190,13 @@ TEST(FL_playXsvf) {
 	fStatus = flPlayXSVF(fx2Handle, "bad.dat", NULL);
 	CHECK_EQUAL(FL_FILE_ERR, fStatus);
 
-	// Verify that a non-existent XSVF file gives FL_FILE_ERR
+	// Verify that a non-existent XSVF file gives FL_BUF_LOAD_ERR
 	fStatus = flPlayXSVF(fx2Handle, "nonExistentFile.xsvf", NULL);
-	CHECK_EQUAL(FL_FILE_ERR, fStatus);
+	CHECK_EQUAL(FL_BUF_LOAD_ERR, fStatus);
 
-	// Verify that a badly-formed XSVF file gives FL_FILE_ERR
+	// Verify that a badly-formed XSVF file gives FL_UNSUPPORTED_CMD_ERR
 	fStatus = flPlayXSVF(fx2Handle, "bad.xsvf", NULL);
-	CHECK_EQUAL(FL_FILE_ERR, fStatus);
+	CHECK_EQUAL(FL_UNSUPPORTED_CMD_ERR, fStatus);
 
 	// Verify that the AVR (having nothing on its JTAG lines) gives FL_JTAG_ERR because the XSVF's IDCODE check fails
 	fStatus = flPlayXSVF(avrHandle, "../gen_xsvf/s3board.xsvf", NULL);

tests-unit/testCsvf.cpp

@@ -17,7 +17,7 @@
 #include <UnitTest++.h>
 #include <makestuff.h>
 #include <libbuffer.h>
-#include "../libfpgalink.h"
+#include "../private.h"
 #include "../xsvf.h"
 #include "../csvfreader.h"
 
@@ -30,22 +30,36 @@ static void testRoundTrip(const char *xsvfFile, uint32 expectedMaxBufSize) {
 	CHECK(bStatus == BUF_SUCCESS);
 	bStatus = bufInitialise(&uncompressedBuf, 1024, 0x00, NULL);
 	CHECK(bStatus == BUF_SUCCESS);
-	fStatus = flLoadXsvfAndConvertToCsvf(xsvfFile, &csvfBuf, &maxBufSize, &uncompressedBuf, NULL);
+
+	// Load XSVF, convert to uncompressed CSVF
+	fStatus = flLoadXsvfAndConvertToCsvf(xsvfFile, &csvfBuf, &maxBufSize, NULL);
 	CHECK(fStatus == FL_SUCCESS);
 	CHECK_EQUAL(expectedMaxBufSize, maxBufSize);
 
+	// Make a copy of the uncompressed buffer
+	bStatus = bufDeepCopy(&uncompressedBuf, &csvfBuf, NULL);
+	CHECK(bStatus == BUF_SUCCESS);
+
+	// Compress the CSVF buffer
+	fStatus = flCompressCsvf(&csvfBuf, NULL);
+	CHECK(fStatus == FL_SUCCESS);
+
+	// Make a reader to iterate over the compressed data
 	Context cp;
 	uint8 thisByte;
-	thisByte = csvfInitReader(&cp, csvfBuf.data);
+	thisByte = csvfInitReader(&cp, csvfBuf.data, true);
 	CHECK(thisByte == 0x00);
 
+	// Uncompress the compressed data into the reconstituteBuf
 	Buffer reconstituteBuf;
 	bStatus = bufInitialise(&reconstituteBuf, uncompressedBuf.length, 0x00, NULL);
 	CHECK(bStatus == BUF_SUCCESS);
 	for ( uint32 i = 0; i < uncompressedBuf.length; i++ ) {
 		bStatus = bufAppendByte(&reconstituteBuf, csvfGetByte(&cp), NULL);
 		CHECK(bStatus == BUF_SUCCESS);
 	}
+
+	// Make sure the result of the compress-uncompress operation is the same as the original data
 	CHECK_EQUAL(uncompressedBuf.length, reconstituteBuf.length);
 	CHECK_ARRAY_EQUAL(uncompressedBuf.data, reconstituteBuf.data, uncompressedBuf.length);