StagingBuff: support both Mlock & IOUring pin; Use in IoUAssmEngn

We use io_uring_register_buffers() for IoUringAssemblyEngine instead
of using mlock(). This __appears__ to have reduced CPU utilization on
the Dell laptop. Could also be that we recently upgraded total RAM
from 8GiB to 32GiB.

commonLibs/attachmentSupport/CMakeLists.txt

@@ -1,3 +1,5 @@
+pkg_check_modules(ATTACHMENT_SUPPORT_URING REQUIRED liburing)
+
 add_library(attachmentSupport SHARED
 	compute.cpp
 	stimulusProducer.cpp
@@ -14,12 +16,21 @@ target_include_directories(attachmentSupport PUBLIC
 	${CMAKE_SOURCE_DIR}/include
 	${CMAKE_BINARY_DIR}/include
 )
+target_include_directories(attachmentSupport PRIVATE
+	${ATTACHMENT_SUPPORT_URING_INCLUDE_DIRS}
+)
 
 target_link_libraries(attachmentSupport PUBLIC
 	Boost::system
 	Boost::log
 	spinscale
 )
+target_link_libraries(attachmentSupport PRIVATE
+	${ATTACHMENT_SUPPORT_URING_LIBRARIES}
+)
+target_link_directories(attachmentSupport PRIVATE
+	${ATTACHMENT_SUPPORT_URING_LIBRARY_DIRS}
+)
 
 # Verify Boost dynamic dependencies after build
 add_custom_command(TARGET attachmentSupport POST_BUILD

commonLibs/attachmentSupport/stagingBuffer.cpp

@@ -1,15 +1,33 @@
 #include <user/stagingBuffer.h>
+#include <cassert>
 #include <unistd.h>
 #include <cstdint>
 #include <stdexcept>
 #include <sys/mman.h>
 #include <vector>
+#include <liburing.h>
 
 #include <user/frameAssemblyDesc.h>
 
 namespace smo {
 namespace stim_buff {
 
+static const char* pinningMechanismToString(
+	StagingBuffer::PinningMechanism mechanism)
+{
+	switch (mechanism)
+	{
+		case StagingBuffer::PinningMechanism::NONE:
+			return "NONE";
+		case StagingBuffer::PinningMechanism::MLOCK:
+			return "MLOCK";
+		case StagingBuffer::PinningMechanism::IO_URING:
+			return "IO_URING";
+	}
+
+	return "Unknown";
+}
+
 // Static defaults for io_uring
 const StagingBuffer::IOEngineConstraints
 StagingBuffer::IOEngineConstraints::ioUringConstraints(
@@ -169,11 +187,9 @@ StagingBuffer::StagingBuffer(
 	const IOEngineConstraints& inputEngineConstraints_,
 	const IOEngineConstraints& /*outputEngineConstraints*/,
 	size_t nSlots)
-: buffer(nullptr, MmapDeleter(0)), bufferNBytes(0),
-nSlots(nSlots), slotStrideNBytes(0),
-firstSlotOffsetNBytes(0),
-inputConstraints(inputEngineConstraints_),
-assemblingFlag(false)
+: buffer(nullptr, MmapDeleter(0)),
+nSlots(nSlots),
+inputConstraints(inputEngineConstraints_)
 {
 	if (nSlots == 0)
 	{
@@ -202,13 +218,6 @@ assemblingFlag(false)
 		static_cast<uint8_t*>(mmapped), MmapDeleter(bufferNBytes));
 	currentNBytes.store(0);
 
-	// Lock the buffer in memory to prevent swapping
-	if (mlock(buffer.get(), bufferNBytes) != 0)
-	{
-		throw std::runtime_error(std::string(__func__)
-			+ ": StagingBuffer: mlock() failed");
-	}
-
 	// Calculate offset and validate invariants (helper function in .cpp)
 	firstSlotOffsetNBytes = StagingBuffer::calculateFirstSlotOffsetAndValidate(
 		buffer.get(), bufferNBytes, nSlots,
@@ -232,5 +241,113 @@ assemblingFlag(false)
 		std::move(slots));
 }
 
+StagingBuffer::~StagingBuffer()
+{
+	assert(!currentlyPinned);
+}
+
+StagingBuffer::Pinner::Pinner(StagingBuffer& parent_)
+: parent(parent_)
+{}
+
+void StagingBuffer::assertUnpinnedAndMarkPinned(PinningMechanism mechanism)
+{
+	if (currentlyPinned)
+	{
+		throw std::runtime_error(
+			std::string(__func__) + ": StagingBuffer already pinned with "
+			+ pinningMechanismToString(currentPinningMechanism));
+	}
+
+	currentlyPinned = true;
+	currentPinningMechanism = mechanism;
+}
+
+std::unique_ptr<StagingBuffer::MlockPinner> StagingBuffer::makeMlockPinner()
+{
+	return std::make_unique<MlockPinner>(*this);
+}
+
+std::unique_ptr<StagingBuffer::IoUringPinner> StagingBuffer::makeIoUringPinner(
+	struct io_uring* ring)
+{
+	return std::make_unique<IoUringPinner>(*this, ring);
+}
+
+StagingBuffer::MlockPinner::MlockPinner(StagingBuffer& parent_)
+: Pinner(parent_)
+{
+	if (!parent.buffer || parent.bufferNBytes == 0)
+	{
+		throw std::runtime_error(std::string(__func__)
+			+ ": Cannot mlock an uninitialized StagingBuffer");
+	}
+
+	parent.assertUnpinnedAndMarkPinned(PinningMechanism::MLOCK);
+
+	if (mlock(parent.buffer.get(), parent.bufferNBytes) != 0)
+	{
+		parent.currentlyPinned = false;
+		parent.currentPinningMechanism = PinningMechanism::NONE;
+		throw std::runtime_error(std::string(__func__)
+			+ ": mlock() failed");
+	}
+}
+
+StagingBuffer::MlockPinner::~MlockPinner()
+{
+	assert(parent.currentlyPinned);
+	assert(parent.currentPinningMechanism == PinningMechanism::MLOCK);
+
+	int ret = munlock(parent.buffer.get(), parent.bufferNBytes);
+	assert(ret == 0);
+	(void)ret;
+
+	parent.currentlyPinned = false;
+	parent.currentPinningMechanism = PinningMechanism::NONE;
+}
+
+StagingBuffer::IoUringPinner::IoUringPinner(
+	StagingBuffer& parent_, struct io_uring* ring_)
+: Pinner(parent_), ring(ring_)
+{
+	if (!ring)
+	{
+		throw std::runtime_error(std::string(__func__)
+			+ ": io_uring ring pointer is null");
+	}
+
+	if (!parent.buffer || parent.bufferNBytes == 0)
+	{
+		throw std::runtime_error(std::string(__func__)
+			+ ": Cannot register an uninitialized StagingBuffer");
+	}
+
+	parent.assertUnpinnedAndMarkPinned(PinningMechanism::IO_URING);
+
+	struct iovec iov = parent.getIoUringRegisterIoVec();
+	int ret = io_uring_register_buffers(ring, &iov, 1);
+	if (ret < 0)
+	{
+		parent.currentlyPinned = false;
+		parent.currentPinningMechanism = PinningMechanism::NONE;
+		throw std::runtime_error(std::string(__func__)
+			+ ": io_uring_register_buffers failed");
+	}
+}
+
+StagingBuffer::IoUringPinner::~IoUringPinner()
+{
+	assert(parent.currentlyPinned);
+	assert(parent.currentPinningMechanism == PinningMechanism::IO_URING);
+
+	int ret = io_uring_unregister_buffers(ring);
+	assert(ret == 0);
+	(void)ret;
+
+	parent.currentlyPinned = false;
+	parent.currentPinningMechanism = PinningMechanism::NONE;
+}
+
 } // namespace stim_buff
 } // namespace smo