made it so the TextBuffer is **actually** type agnostic.

this required improving how we render control chars
and refactoring the X selection update callback.

This will allow things like
a) unicode char to be stored
b) the style buffer to be of uint8_t instead of chars and casted

src/DocumentWidget.cpp

@@ -588,6 +588,7 @@ DocumentWidget::DocumentWidget(const QString &name, QWidget *parent, Qt::WindowF
 	// Every document has a backing buffer
 	info_->buffer = std::make_shared<TextBuffer>();
 	info_->buffer->BufAddModifyCB(Highlight::SyntaxHighlightModifyCB, this);
+	info_->buffer->BufSetSelectionUpdate(TextArea::updatePrimarySelection);
 
 	// create the text widget
 	if (Settings::splitHorizontally) {
@@ -6262,7 +6263,6 @@ std::unique_ptr<WindowHighlightData> DocumentWidget::createHighlightData(Pattern
 
 	// Create the style buffer
 	auto styleBuf = std::make_unique<TextBuffer>();
-	styleBuf->BufSetSyncXSelection(false);
 
 	const int contextLines = patternSet->lineContext;
 	const int contextChars = patternSet->charContext;

src/MainWindow.cpp

@@ -6921,7 +6921,6 @@ void MainWindow::replaceInSelection(DocumentWidget *document, TextArea *area, co
 	   intermediate steps from the display routines, and so everything can
 	   be undone in a single operation */
 	TextBuffer tempBuf;
-	tempBuf.BufSetSyncXSelection(false);
 	tempBuf.BufSetAll(fileString);
 
 	// search the string and do the replacements in the temporary buffer

src/TextArea.cpp

@@ -2008,7 +2008,6 @@ void TextArea::findWrapRange(view::string_view deletedText, TextCursor pos, int6
 	const int64_t length = (pos - countFrom) + nDeleted + (countTo - (pos + nInserted));
 
 	TextBuffer deletedTextBuf;
-	deletedTextBuf.BufSetSyncXSelection(false);
 
 	if (pos > countFrom) {
 		deletedTextBuf.BufCopyFromBuf(buffer_, countFrom, pos, buffer_->BufStartOfBuffer());
@@ -4307,7 +4306,6 @@ std::string TextArea::wrapText(view::string_view startLine, view::string_view te
 
 	// Create a temporary text buffer and load it with the strings
 	TextBuffer wrapBuf;
-	wrapBuf.BufSetSyncXSelection(false);
 	wrapBuf.BufInsert(buffer_->BufStartOfBuffer(), startLine);
 	wrapBuf.BufAppend(text);
 
@@ -6081,7 +6079,6 @@ void TextArea::beginBlockDrag() {
 	/* Save a copy of the whole text buffer as a backup, and for
 	   deriving changes */
 	dragOrigBuf_ = std::make_unique<TextBuffer>();
-	dragOrigBuf_->BufSetSyncXSelection(false);
 	dragOrigBuf_->BufSetTabDistance(buffer_->BufGetTabDistance(), true);
 	dragOrigBuf_->BufSetUseTabs(buffer_->BufGetUseTabs());
 
@@ -6119,7 +6116,6 @@ void TextArea::beginBlockDrag() {
 	dragInserted_  = sel.end() - sel.start();
 	if (sel.isRectangular()) {
 		TextBuffer testBuf;
-		testBuf.BufSetSyncXSelection(false);
 
 		std::string testText = buffer_->BufGetRange(sel.start(), sel.end());
 		testBuf.BufSetTabDistance(buffer_->BufGetTabDistance(), true);
@@ -6217,7 +6213,6 @@ void TextArea::blockDragSelection(const QPoint &pos, BlockDragTypes dragType) {
 	   range of characters which might be modified in this drag step
 	   (this could be tighter, but hopefully it's not too slow) */
 	TextBuffer tempBuf;
-	tempBuf.BufSetSyncXSelection(false);
 	tempBuf.BufSetTabDistance(buffer_->BufGetTabDistance(), false);
 	tempBuf.BufSetUseTabs(buffer_->BufGetUseTabs());
 
@@ -7356,7 +7351,6 @@ std::string TextArea::TextGetWrapped(TextCursor startPos, TextCursor endPos) {
 	   newlines will expand it to.  Since it's a text buffer, if we guess
 	   wrong, it will fail softly, and simply expand the size */
 	TextBuffer outBuf((endPos - startPos) + (endPos - startPos) / 5);
-	outBuf.BufSetSyncXSelection(false);
 
 	TextCursor outPos;
 
@@ -8040,3 +8034,21 @@ int TextArea::fixedFontHeight() const {
 int TextArea::fixedFontWidth() const {
 	return fixedFontWidth_;
 }
+
+void TextArea::updatePrimarySelection(const std::shared_ptr<TextBuffer> &buffer) {
+#ifdef Q_OS_UNIX
+	if (QApplication::clipboard()->supportsSelection()) {
+		const bool selected = buffer->primary.selected_;
+		const bool isOwner  = TextAreaMimeData::isOwner(QApplication::clipboard()->mimeData(QClipboard::Selection), buffer.get());
+
+		// if we already own the selection, then we don't need to do anything
+		// things are lazily evaluated in TextAreaMimeData::retrieveData
+		if ((isOwner && selected) || (!isOwner && !selected)) {
+			return;
+		}
+
+		auto data = new TextAreaMimeData(buffer);
+		QApplication::clipboard()->setMimeData(data, QClipboard::Selection);
+	}
+#endif
+}

src/TextArea.h

@@ -90,6 +90,9 @@ class TextArea final : public QAbstractScrollArea {
 	TextArea &operator=(const TextArea &) = delete;
 	~TextArea() override;
 
+public:
+	static void updatePrimarySelection(const std::shared_ptr<TextBuffer> &buffer);
+
 public:
 	// NOTE(eteran): if these aren't expected to have side effects, then some
 	// of them may be able to be replaced with signals

src/TextBuffer.h

@@ -32,15 +32,18 @@ class BasicTextBuffer : public std::enable_shared_from_this<BasicTextBuffer<Ch,
 	using view_type   = view::basic_string_view<Ch, Tr>;
 
 public:
-	using modify_callback_type     = void (*)(TextCursor pos, int64_t nInserted, int64_t nDeleted, int64_t nRestyled, view_type deletedText, void *user);
-	using pre_delete_callback_type = void (*)(TextCursor pos, int64_t nDeleted, void *user);
+	using modify_callback_type           = void (*)(TextCursor pos, int64_t nInserted, int64_t nDeleted, int64_t nRestyled, view_type deletedText, void *user);
+	using pre_delete_callback_type       = void (*)(TextCursor pos, int64_t nDeleted, void *user);
+	using selection_update_callback_type = void (*)(const std::shared_ptr<BasicTextBuffer<Ch, Tr>> &);
 
 private:
 	/* Initial size for the buffer gap (empty space in the buffer where text
 	 * might be inserted if the user is typing sequential chars)
 	 */
 	static constexpr int PreferredGapSize = 80;
 
+	static const char *ControlCodeTable[32];
+
 public:
 	/* Maximum length in characters of a tab or control character expansion
 	 * of a single buffer character
@@ -99,6 +102,8 @@ class BasicTextBuffer : public std::enable_shared_from_this<BasicTextBuffer<Ch,
 	Ch back() const noexcept;
 
 public:
+	selection_update_callback_type BufGetSelectionUpdate() const;
+	void BufSetSelectionUpdate(selection_update_callback_type fn);
 	bool BufGetEmptySelectionPos(TextCursor *start, TextCursor *end, bool *isRect, int64_t *rectStart, int64_t *rectEnd) const noexcept;
 	bool BufGetSelectionPos(TextCursor *start, TextCursor *end, bool *isRect, int64_t *rectStart, int64_t *rectEnd) const noexcept;
 	boost::optional<SelectionPos> BufGetSelectionPos() const noexcept;
@@ -204,6 +209,7 @@ class BasicTextBuffer : public std::enable_shared_from_this<BasicTextBuffer<Ch,
 	static int textWidth(view_type text, int tabDist) noexcept;
 	static int64_t countLines(view_type string) noexcept;
 	static void overlayRectInLine(view_type line, view_type insLine, int64_t rectStart, int64_t rectEnd, int tabDist, bool useTabs, string_type *outStr, int64_t *endOffset) noexcept;
+	static int writeControl(Ch out[20], const char *ctrl_char) noexcept;
 
 private:
 	template <class Out>
@@ -224,6 +230,7 @@ class BasicTextBuffer : public std::enable_shared_from_this<BasicTextBuffer<Ch,
 private:
 	std::deque<std::pair<pre_delete_callback_type, void *>> preDeleteProcs_; // procedures to call before text is deleted from the buffer; at most one is supported.
 	std::deque<std::pair<modify_callback_type, void *>> modifyProcs_;        // procedures to call when buffer is modified to redisplay contents
+	selection_update_callback_type selectionUpdate_ = nullptr;
 
 public:
 	Selection primary; // highlighted areas

src/TextBuffer.tcc

@@ -2,47 +2,18 @@
 #ifndef TEXT_BUFFER_TCC_
 #define TEXT_BUFFER_TCC_
 
-#include "TextAreaMimeData.h"
 #include "TextBuffer.h"
 #include "Util/algorithm.h"
-
 #include <algorithm>
 #include <cassert>
+#include <cstring>
 
-#include <QApplication>
-#include <QClipboard>
-#include <QtDebug>
-
-namespace detail {
-
-template <class Ch>
-const Ch *controlCharacter(size_t index) noexcept;
-
-template <>
-constexpr const char *controlCharacter<char>(size_t index) noexcept {
-	const char *const ControlCodeTable[32] = {
-		"nul", "soh", "stx", "etx", "eot", "enq", "ack", "bel",
-		"bs", "ht", "nl", "vt", "np", "cr", "so", "si",
-		"dle", "dc1", "dc2", "dc3", "dc4", "nak", "syn", "etb",
-		"can", "em", "sub", "esc", "fs", "gs", "rs", "us"};
-
-	assert(index < 32);
-	return ControlCodeTable[index];
-}
-
-template <>
-constexpr const wchar_t *controlCharacter<wchar_t>(size_t index) noexcept {
-	const wchar_t *const ControlCodeTable[32] = {
-		L"nul", L"soh", L"stx", L"etx", L"eot", L"enq", L"ack", L"bel",
-		L"bs", L"ht", L"nl", L"vt", L"np", L"cr", L"so", L"si",
-		L"dle", L"dc1", L"dc2", L"dc3", L"dc4", L"nak", L"syn", L"etb",
-		L"can", L"em", L"sub", L"esc", L"fs", L"gs", L"rs", L"us"};
-
-	assert(index < 32);
-	return ControlCodeTable[index];
-}
-
-}
+template <class Ch, class Tr>
+const char *BasicTextBuffer<Ch, Tr>::ControlCodeTable[32] = {
+	"nul", "soh", "stx", "etx", "eot", "enq", "ack", "bel",
+	"bs", "ht", "nl", "vt", "np", "cr", "so", "si",
+	"dle", "dc1", "dc2", "dc3", "dc4", "nak", "syn", "etb",
+	"can", "em", "sub", "esc", "fs", "gs", "rs", "us"};
 
 /*
 ** Get the entire contents of a text buffer.
@@ -569,21 +540,9 @@ void BasicTextBuffer<Ch, Tr>::BufSelect(std::pair<TextCursor, TextCursor> range)
 
 template <class Ch, class Tr>
 void BasicTextBuffer<Ch, Tr>::updatePrimarySelection() noexcept {
-#ifdef Q_OS_UNIX
-	if (syncXSelection_ && QApplication::clipboard()->supportsSelection()) {
-		const bool selected = primary.selected_;
-		const bool isOwner  = TextAreaMimeData::isOwner(QApplication::clipboard()->mimeData(QClipboard::Selection), this);
-
-		// if we already own the selection, then we don't need to do anything
-		// things are lazily evaluated in TextAreaMimeData::retrieveData
-		if ((isOwner && selected) || (!isOwner && !selected)) {
-			return;
-		}
-
-		auto data = new TextAreaMimeData(this->shared_from_this());
-		QApplication::clipboard()->setMimeData(data, QClipboard::Selection);
+	if (syncXSelection_ && selectionUpdate_) {
+		selectionUpdate_(this->shared_from_this());
 	}
-#endif
 }
 
 template <class Ch, class Tr>
@@ -841,6 +800,21 @@ int BasicTextBuffer<Ch, Tr>::BufExpandTab(int64_t indent, Ch outStr[MAX_EXP_CHAR
 	return nSpaces;
 }
 
+template <class Ch, class Tr>
+int BasicTextBuffer<Ch, Tr>::writeControl(Ch out[MAX_EXP_CHAR_LEN], const char *ctrl_char) noexcept {
+
+	auto ptr = out;
+	auto in  = ctrl_char;
+
+	*ptr++ = '<';
+	while (const char ch = *in++) {
+		*ptr++ = Ch(ch);
+	}
+	*ptr++ = '>';
+
+	return ptr - out;
+}
+
 template <class Ch, class Tr>
 int BasicTextBuffer<Ch, Tr>::BufExpandCharacter(Ch ch, int64_t indent, Ch outStr[MAX_EXP_CHAR_LEN], int tabDist) noexcept {
 
@@ -851,12 +825,13 @@ int BasicTextBuffer<Ch, Tr>::BufExpandCharacter(Ch ch, int64_t indent, Ch outStr
 
 #if defined(VISUAL_CTRL_CHARS)
 	// Convert ASCII control codes to readable character sequences
-	if ((static_cast<size_t>(ch)) < 32) {
-		return snprintf(outStr, MAX_EXP_CHAR_LEN, "<%s>", detail::controlCharacter<Ch>(static_cast<size_t>(ch)));
+	if (static_cast<size_t>(ch) < 32) {
+		const char *s = ControlCodeTable[static_cast<size_t>(ch)];
+		return writeControl(outStr, s);
 	}
 
 	if (ch == 127) {
-		return snprintf(outStr, MAX_EXP_CHAR_LEN, "<del>");
+		return writeControl(outStr, "del");
 	}
 #endif
 	// Otherwise, just return the character
@@ -877,12 +852,12 @@ int BasicTextBuffer<Ch, Tr>::BufCharWidth(Ch ch, int64_t indent, int tabDist) no
 
 #if defined(VISUAL_CTRL_CHARS)
 	if (static_cast<size_t>(ch) < 32) {
-		const Ch *const s = detail::controlCharacter<Ch>(static_cast<size_t>(ch));
-		return static_cast<int>(Tr::length(s) + 2);
+		const char *s = ControlCodeTable[static_cast<size_t>(ch)];
+		return static_cast<int>(strlen(s) + 2);
 	}
 
 	if (ch == 127) {
-		return 5; // Tr::length("<del>")
+		return 5; // strlen("<del>")
 	}
 #endif
 	return 1;
@@ -2126,6 +2101,16 @@ bool BasicTextBuffer<Ch, Tr>::BufSetSyncXSelection(bool sync) {
 	return std::exchange(syncXSelection_, sync);
 }
 
+template <class Ch, class Tr>
+auto BasicTextBuffer<Ch, Tr>::BufGetSelectionUpdate() const -> selection_update_callback_type {
+	return selectionUpdate_;
+}
+
+template <class Ch, class Tr>
+void BasicTextBuffer<Ch, Tr>::BufSetSelectionUpdate(selection_update_callback_type fn) {
+	selectionUpdate_ = fn;
+}
+
 /**
  * @brief TextSelection::setSelection
  * @param newStart

src/shift.cpp

@@ -218,7 +218,6 @@ std::string fillParagraphs(view::string_view text, int64_t rightMargin, int tabD
 
 	// Create a buffer to accumulate the filled paragraphs
 	TextBuffer buf;
-	buf.BufSetSyncXSelection(false);
 	buf.BufSetAll(text);
 
 	/*
@@ -618,7 +617,6 @@ void shiftRect(DocumentWidget *document, TextArea *area, ShiftDirection directio
 	/* Create a temporary buffer for the lines containing the selection, to
 	   hide the intermediate steps from the display update routines */
 	TextBuffer tempBuf;
-	tempBuf.BufSetSyncXSelection(false);
 	tempBuf.BufSetTabDistance(buf->BufGetTabDistance(), false);
 	tempBuf.BufSetUseTabs(buf->BufGetUseTabs());