Add QutexAcquisitionHistoryTracker; integrate plumbing

We add the new Qutex acquisision history tracker that allows us
to dynamically detect qutex gridlocks. We've integrated it into
LockerAndInvoker::operator() in a preliminary way.

We also moved all of the trace*ForGridlockOn() methods into the
new QutexAcquisitionHistoryTracker singleton class. They're
more appropriately located there. They're still unimplemented
though.

include/asynchronousContinuationChainLink.h

@@ -13,8 +13,12 @@ namespace smo {
  *
  * The chain walking logic can use dynamic_cast to determine the most
  * derived type and perform appropriate operations.
+ *
+ * Inherits from enable_shared_from_this to allow objects to obtain a
+ * shared_ptr to themselves, which is useful for gridlock detection tracking.
  */
 class AsynchronousContinuationChainLink
+:	public std::enable_shared_from_this<AsynchronousContinuationChainLink>
 {
 public:
     virtual ~AsynchronousContinuationChainLink() = default;

include/qutexAcquisitionHistoryTracker.h

@@ -0,0 +1,126 @@
+#ifndef QUTEX_ACQUISITION_HISTORY_TRACKER_H
+#define QUTEX_ACQUISITION_HISTORY_TRACKER_H
+
+#include <unordered_map>
+#include <memory>
+#include <forward_list>
+#include <functional>
+
+
+namespace smo {
+
+// Forward declarations
+class Qutex;
+class AsynchronousContinuationChainLink;
+
+/**
+ * @brief QutexAcquisitionHistoryTracker - Tracks acquisition history for
+ *        gridlock detection
+ *
+ * This class maintains a central acquisition history to track all lockvokers
+ * suspected of being gridlocked. It stores information about what locks each
+ * timed-out lockvoker wants and what locks they hold in their continuation
+ * history.
+ */
+class QutexAcquisitionHistoryTracker
+{
+public:
+	/**
+	 * @brief Type definition for the acquisition history entry
+	 *
+	 * pair.first: The firstFailedQutex that this lockvoker WANTS but can't
+	 * acquire
+	 * pair.second: A unique_ptr to a list of all acquired Qutexes in this
+	 * lockvoker's continuation history
+	 */
+	typedef std::pair<
+		std::reference_wrapper<Qutex>,
+		std::unique_ptr<std::forward_list<std::reference_wrapper<Qutex>>>
+	> AcquisitionHistoryEntry;
+
+	/**
+	 * @brief Type definition for the acquisition history map
+	 *
+	 * Key: std::shared_ptr<AsynchronousContinuationChainLink>
+	 *		(the continuation that contains the timed-out lockvoker)
+	 * Value: AcquisitionHistoryEntry
+	 *		(its wanted lock (aka: firstFailedQutex/pair.first) + held locks)
+	 */
+	typedef std::unordered_map<
+		std::shared_ptr<AsynchronousContinuationChainLink>,
+		AcquisitionHistoryEntry
+	> AcquisitionHistoryMap;
+
+public:
+	static QutexAcquisitionHistoryTracker& getInstance()
+	{
+		static QutexAcquisitionHistoryTracker instance;
+		return instance;
+	}
+
+	/**
+	 * @brief Add a continuation to the acquisition history if it doesn't
+	 *	already exist
+	 * @param continuation Shared pointer to the
+	 *	AsynchronousContinuationChainLink
+	 * @param wantedLock The lock that this continuation wants but can't
+	 *	acquire
+	 * @param heldLocks Unique pointer to list of locks held in this
+	 *	continuation's history (will be moved)
+	 */
+	void addIfNotExists(
+		std::shared_ptr<AsynchronousContinuationChainLink> &continuation,
+		Qutex& wantedLock,
+		std::unique_ptr<std::forward_list<std::reference_wrapper<Qutex>>>
+			heldLocks
+		)
+	{
+		auto it = acquisitionHistory.find(continuation);
+
+		// If a continuation already exists, don't add it again
+		if (it != acquisitionHistory.end()) {
+			return;
+		}
+
+		acquisitionHistory.emplace(continuation, std::make_pair(
+			std::ref(wantedLock), std::move(heldLocks)));
+	}
+
+	/**
+	 * @brief Remove a continuation from the acquisition history
+	 * @param continuation Shared pointer to the AsynchronousContinuationChainLink
+	 *	to remove
+	 * @return true if the continuation was found and removed, false if not found
+	 */
+	bool remove(std::shared_ptr<AsynchronousContinuationChainLink> &continuation)
+	{
+		auto it = acquisitionHistory.find(continuation);
+		if (it != acquisitionHistory.end()) {
+			acquisitionHistory.erase(it);
+			return true;
+		}
+		return false;
+	}
+
+	bool heuristicallyTraceContinuationHistoryForGridlockOn(
+		Qutex &firstFailedQutex) const;
+	bool completelyTraceContinuationHistoryForGridlockOn(
+		Qutex &firstFailedQutex) const;
+
+	// Disable copy constructor and assignment operator
+	QutexAcquisitionHistoryTracker(
+		const QutexAcquisitionHistoryTracker&) = delete;
+	QutexAcquisitionHistoryTracker& operator=(
+		const QutexAcquisitionHistoryTracker&) = delete;
+
+private:
+	QutexAcquisitionHistoryTracker() = default;
+	~QutexAcquisitionHistoryTracker() = default;
+
+private:
+	AcquisitionHistoryMap acquisitionHistory;
+};
+
+} // namespace smo
+
+#endif // QUTEX_ACQUISITION_HISTORY_TRACKER_H

include/serializedAsynchronousContinuation.h

@@ -12,6 +12,7 @@
 #include <asynchronousContinuation.h>
 #include <lockerAndInvokerBase.h>
 #include <callback.h>
+#include <qutexAcquisitionHistoryTracker.h>
 
 namespace smo {
 
@@ -36,6 +37,10 @@ public:
 			std::forward<Args>(args)...);
 	}
 
+	// Return list of all qutexes in predecessors' LockSets; excludes self.
+	std::unique_ptr<std::forward_list<std::reference_wrapper<Qutex>>>
+	getAcquiredQutexHistory() const;
+
 public:
 	LockSet<OriginalCbFnT> requiredLocks;
 	std::atomic<bool> isAwakeOrBeingAwakened{false};
@@ -116,8 +121,33 @@ public:
 				bool isDeadlock = traceContinuationHistoryForDeadlockOn(
 					firstFailedQutex);
 
-				bool isGridlock = heuristicallyTraceContinuationHistoryForGridlockOn(
-					firstFailedQutex);
+				bool gridlockIsHeuristicallyLikely = false;
+				bool gridlockIsAlgorithmicallyLikely = false;
+
+				if (gridlockLikely)
+				{
+					auto tracker = QutexAcquisitionHistoryTracker
+						::getInstance();
+
+					auto heldLocks = serializedContinuation
+						.getAcquiredQutexHistory();
+
+					// Add this continuation to the tracker
+					tracker.addIfNotExists(
+						serializedContinuation.shared_from_this(),
+						firstFailedQutex, std::move(heldLocks));
+
+					gridlockIsHeuristicallyLikely = tracker
+						.heuristicallyTraceContinuationHistoryForGridlockOn(
+							firstFailedQutex);
+
+					gridlockIsAlgorithmicallyLikely = tracker
+						.completelyTraceContinuationHistoryForGridlockOn(
+							firstFailedQutex);
+				}
+
+				bool isGridlock = (gridlockIsHeuristicallyLikely
+					|| gridlockIsAlgorithmicallyLikely);
 
 				if (!isDeadlock && !isGridlock)
 					{ return; }
@@ -145,6 +175,28 @@ public:
 			 * can't acquire the locks anyway.
 			 */
 			serializedContinuation.requiredLocks.unregisterFromQutexQueues();
+
+#ifdef CONFIG_ENABLE_DEBUG_LOCKS
+			/**	EXPLANATION:
+			 * If we were being tracked for gridlock detection but successfully
+			 * acquired all locks, it was a false positive due to timed delay,
+			 * long-running operation, or I/O delay
+			 */
+			if (gridlockLikely)
+			{
+				bool removed = QutexAcquisitionHistoryTracker::getInstance()
+					.remove(serializedContinuation.shared_from_this());
+
+				if (removed)
+				{
+					std::cerr << "LockerAndInvoker::operator(): False positive gridlock "
+						"detection - continuation was being tracked but successfully "
+						"acquired all locks. This was likely due to timed delay, "
+						"long-running operation, or I/O delay." << std::endl;
+				}
+			}
+#endif
+
 			invocationTarget();
 		}
 
@@ -227,10 +279,6 @@ public:
 		};
 
 		bool traceContinuationHistoryForDeadlockOn(Qutex &firstFailedQutex);
-		bool heuristicallyTraceContinuationHistoryForGridlockOn(
-			Qutex &firstFailedQutex);
-		bool completelyTraceContinuationHistoryForGridlockOn(
-			Qutex &firstFailedQutex);
 		bool traceContinuationHistoryForDeadlock(void)
 		{
 			for (auto& lockUsageDesc