Split CompThread=>MindThr+MrnttThr; alloc globalMind in mrnttMain

We now allocate globalMind locally inside of marionetteMain. Why?

Before now, we had an asymmetric threading situation where the
globalMind's threads were initialized at during global constructor
invocation and not on demand. This meant that we had to shut down
those threads even if we had never got to the point of calling
Mind::initializeReq.

This significantly complicated our shutdown sequence since we had
to factor in the lifetime of the std::thread objects inside of the
ComponentThreads which were inside of the globalMind object.

Now, if we hadn't called Mind::initializeReq, we don't have to
perform any Mind::finalizeReq or adjacent operations. Shutdown is
symmetrically mirrored against the operations we actually performed
during execution.

We introduced some complexity by splitting ComponentThreads into
two derivative types (MindThread and MarionetteThread) but I think
in the long term we'll be able to massage this split into a much
cleaner situation overall.

smocore/componentThread.cpp

@@ -13,17 +13,18 @@ namespace smo {
 
 thread_local std::shared_ptr<ComponentThread> thisComponentThread;
 
-namespace mrntt {
-extern std::shared_ptr<ComponentThread> mrntt;
-}
-
 // Implementation of static method
-std::shared_ptr<ComponentThread> ComponentThread::getMrntt()
+std::shared_ptr<MarionetteThread> ComponentThread::getMrntt()
 {
 	return mrntt::mrntt;
 }
 
-void ComponentThread::initializeTls(void)
+void MarionetteThread::initializeTls(void)
+{
+	thisComponentThread = shared_from_this();
+}
+
+void MindThread::initializeTls(void)
 {
 	thisComponentThread = shared_from_this();
 }
@@ -39,7 +40,7 @@ const std::shared_ptr<ComponentThread> ComponentThread::getSelf(void)
 	return thisComponentThread;
 }
 
-void ComponentThread::main(ComponentThread& self)
+void MindThread::main(MindThread& self)
 {
 
 	if (OptionParser::getOptions().verbose)
@@ -98,13 +99,13 @@ void ComponentThread::main(ComponentThread& self)
 	std::cout << self.name << ":" << __func__ << ": Exited event loop" << "\n";
 }
 
-class ComponentThread::ThreadLifetimeMgmtOp
+class MindThread::ThreadLifetimeMgmtOp
 :	public TargetedAsynchronousContinuation<threadLifetimeMgmtOpCbFn>
 {
 public:
 	ThreadLifetimeMgmtOp(
 		const std::shared_ptr<ComponentThread> &caller,
-		const std::shared_ptr<ComponentThread> &target,
+		const std::shared_ptr<MindThread> &target,
 		threadLifetimeMgmtOpCbFn callback)
 	:	TargetedAsynchronousContinuation<threadLifetimeMgmtOpCbFn>(
 			caller, callback),
@@ -119,7 +120,7 @@ public:
 	}
 
 public:
-	const std::shared_ptr<ComponentThread> target;
+	const std::shared_ptr<MindThread> target;
 
 public:
 	void joltThreadReq1(
@@ -204,7 +205,7 @@ void ComponentThread::cleanup(void)
 	this->keepLooping = false;
 }
 
-void ComponentThread::joltThreadReq(threadLifetimeMgmtOpCbFn callback)
+void MindThread::joltThreadReq(threadLifetimeMgmtOpCbFn callback)
 {
 	/**	EXPLANATION:
 	 * We can't use shared_from_this() here because JOLTing occurs prior to
@@ -227,9 +228,8 @@ void ComponentThread::joltThreadReq(threadLifetimeMgmtOpCbFn callback)
 			+ ": invoked on mrntt thread");
 	}
 
-	std::shared_ptr<ComponentThread>
-		mrntt = mrntt::mrntt,
-		target = parent.getComponentThread(id);
+	std::shared_ptr<MarionetteThread> mrntt = mrntt::mrntt;
+	std::shared_ptr<MindThread> target = getParent().getComponentThread(id);
 
 	auto request = std::make_shared<ThreadLifetimeMgmtOp>(
 		mrntt, target, callback);
@@ -241,7 +241,7 @@ void ComponentThread::joltThreadReq(threadLifetimeMgmtOpCbFn callback)
 }
 
 // Thread management method implementations
-void ComponentThread::startThreadReq(threadLifetimeMgmtOpCbFn callback)
+void MindThread::startThreadReq(threadLifetimeMgmtOpCbFn callback)
 {
 	std::shared_ptr<ComponentThread> caller = getSelf();
 	auto request = std::make_shared<ThreadLifetimeMgmtOp>(
@@ -253,7 +253,7 @@ void ComponentThread::startThreadReq(threadLifetimeMgmtOpCbFn callback)
 			request.get(), request));
 }
 
-void ComponentThread::exitThreadReq(threadLifetimeMgmtOpCbFn callback)
+void MindThread::exitThreadReq(threadLifetimeMgmtOpCbFn callback)
 {
 	std::shared_ptr<ComponentThread> caller = getSelf();
 	auto request = std::make_shared<ThreadLifetimeMgmtOp>(
@@ -264,13 +264,13 @@ void ComponentThread::exitThreadReq(threadLifetimeMgmtOpCbFn callback)
 			&ThreadLifetimeMgmtOp::exitThreadReq1_mainQueue,
 			request.get(), request));
 
-	this->pause_io_service.post(
+	pause_io_service.post(
 		std::bind(
 			&ThreadLifetimeMgmtOp::exitThreadReq1_pauseQueue,
 			request.get(), request));
 }
 
-void ComponentThread::pauseThreadReq(threadLifetimeMgmtOpCbFn callback)
+void MindThread::pauseThreadReq(threadLifetimeMgmtOpCbFn callback)
 {
 	if (id == ComponentThread::MRNTT)
 	{
@@ -288,7 +288,7 @@ void ComponentThread::pauseThreadReq(threadLifetimeMgmtOpCbFn callback)
 			request.get(), request));
 }
 
-void ComponentThread::resumeThreadReq(threadLifetimeMgmtOpCbFn callback)
+void MindThread::resumeThreadReq(threadLifetimeMgmtOpCbFn callback)
 {
 	if (id == ComponentThread::MRNTT)
 	{
@@ -301,13 +301,13 @@ void ComponentThread::resumeThreadReq(threadLifetimeMgmtOpCbFn callback)
 	auto request = std::make_shared<ThreadLifetimeMgmtOp>(
 		caller, shared_from_this(), callback);
 
-	this->pause_io_service.post(
+	pause_io_service.post(
 		std::bind(
 			&ThreadLifetimeMgmtOp::resumeThreadReq1,
 			request.get(), request));
 }
 
-class ComponentThread::MindShutdownIndOp
+class MindThread::MindShutdownIndOp
 :	public TargetedAsynchronousContinuation<mindShutdownIndOpCbFn>
 {
 public:
@@ -391,13 +391,13 @@ void ComponentThread::exceptionInd(
 			+ ": invoked on non-mrntt thread " + faultyThread->name);
 	}
 
-	auto request = std::make_shared<MindShutdownIndOp>(
+	auto request = std::make_shared<MindThread::MindShutdownIndOp>(
 		faultyThread, nullptr);
 
 	// Post the exception to the mrntt thread.
 	this->getIoService().post(
 		std::bind(
-			&MindShutdownIndOp::mindShutdownInd1_exception,
+			&MindThread::MindShutdownIndOp::mindShutdownInd1_exception,
 			request.get(), request));
 }
 
@@ -409,13 +409,13 @@ void ComponentThread::userShutdownInd()
             + ": invoked on non-mrntt thread " + this->name);
     }
 
-	auto request = std::make_shared<MindShutdownIndOp>(
+	auto request = std::make_shared<MindThread::MindShutdownIndOp>(
 		ComponentThread::getMrntt(), nullptr);
 
     // Post the user shutdown to the mrntt thread.
     this->getIoService().post(
 		std::bind(
-			&MindShutdownIndOp::mindShutdownInd1_userShutdown,
+			&MindThread::MindShutdownIndOp::mindShutdownInd1_userShutdown,
 			request.get(), request));
 }
 
@@ -442,7 +442,7 @@ int ComponentThread::getAvailableCpuCount()
 	return cpuCount;
 }
 
-void ComponentThread::pinToCpu(int cpuId)
+void MindThread::pinToCpu(int cpuId)
 {
 	if (cpuId < 0)
 	{

smocore/include/componentThread.h

@@ -16,9 +16,10 @@
 namespace smo {
 
 class Mind; // Forward declaration
+class MarionetteThread;
+class MindThread;
 
 class ComponentThread
-:   public std::enable_shared_from_this<ComponentThread>
 {
 public:
 	enum ThreadId
@@ -32,57 +33,26 @@ public:
 		N_ITEMS
 	};
 
-	ComponentThread(ThreadId _id, Mind& parent)
-    :   id(_id), name(getThreadName(_id)), parent(parent),
-        work(io_service), pause_work(pause_io_service),
-        pinnedCpuId(-1),
-        thread(
-			((id == MRNTT) ? marionetteMain : main),
-			std::ref(*this))
+protected:
+	ComponentThread(ThreadId _id)
+    :   id(_id), name(getThreadName(_id)),
+        work(io_service)
 	{}
 
+public:
+	virtual ~ComponentThread() = default;
+
 	void cleanup(void);
 
 	boost::asio::io_service& getIoService(void) { return io_service; }
 
-	void initializeTls(void);
 	static const std::shared_ptr<ComponentThread> getSelf(void);
-
-	static std::shared_ptr<ComponentThread> getMrntt();
-	Mind& getParent() const { return parent; }
+	static std::shared_ptr<MarionetteThread> getMrntt();
 
 	typedef void (mainFn)(ComponentThread &self);
-	static mainFn main, marionetteMain;
-
-	typedef std::function<void()> threadLifetimeMgmtOpCbFn;
-	// Thread management methods
-	void startThreadReq(threadLifetimeMgmtOpCbFn callback);
-	void exitThreadReq(threadLifetimeMgmtOpCbFn callback);
-	void pauseThreadReq(threadLifetimeMgmtOpCbFn callback);
-	void resumeThreadReq(threadLifetimeMgmtOpCbFn callback);
-
-	/**
-	 * JOLTs this thread to begin processing after global initialization.
-	 *
-	 * JOLTing is the mechanism that allows threads to enter their main
-	 * event loops and set up TLS vars after all global constructors have
-	 * completed. This prevents race conditions during system startup.
-	 */
-	void joltThreadReq(threadLifetimeMgmtOpCbFn callback);
 
 	// CPU management methods
 	static int getAvailableCpuCount();
-	void pinToCpu(int cpuId);
-
-	enum class ThreadOp
-	{
-		START,
-		PAUSE,
-		RESUME,
-		EXIT,
-		JOLT,
-		N_ITEMS
-	};
 
 	typedef std::function<void()> mindShutdownIndOpCbFn;
 	// Intentionally doesn't take a callback.
@@ -93,19 +63,9 @@ public:
 public:
 	ThreadId id;
 	std::string name;
-	Mind &parent;
 	boost::asio::io_service io_service;
 	boost::asio::io_service::work work;
-	boost::asio::io_service pause_io_service;
-	boost::asio::io_service::work pause_work;
 	std::atomic<bool> keepLooping;
-	int pinnedCpuId;
-
-
-    /* Always ensure that this is last so that the thread is spawned after
-     * everything else is constructed.
-     */
-	std::thread thread;
 
 	static const std::string getThreadName(ThreadId id)
 	{
@@ -127,14 +87,87 @@ public:
 
 		return threadNames[id];
 	}
+};
 
-private:
+class MarionetteThread
+:	public std::enable_shared_from_this<MarionetteThread>,
+	public ComponentThread
+{
+public:
+	MarionetteThread()
+	:	ComponentThread(MRNTT),
+	thread(main, std::ref(*this))
+	{
+	}
+
+	static void main(MarionetteThread& self);
+	void initializeTls(void);
+
+public:
+	std::thread thread;
+};
+
+class MindThread
+:	public std::enable_shared_from_this<MindThread>, public ComponentThread
+{
+public:
+	enum class ThreadOp
+	{
+		START,
+		PAUSE,
+		RESUME,
+		EXIT,
+		JOLT,
+		N_ITEMS
+	};
+
+	MindThread(ThreadId _id, Mind& parent)
+	:   ComponentThread(_id),
+	pinnedCpuId(-1),
+	pause_work(pause_io_service),
+	parent(parent),
+	thread(main, std::ref(*this))
+	{
+	}
+
+	static void main(MindThread& self);
+	void initializeTls(void);
+
+	Mind& getParent() const { return parent; }
+
+	// Thread management methods
+	typedef std::function<void()> threadLifetimeMgmtOpCbFn;
+	void startThreadReq(threadLifetimeMgmtOpCbFn callback);
+	void exitThreadReq(threadLifetimeMgmtOpCbFn callback);
+	void pauseThreadReq(threadLifetimeMgmtOpCbFn callback);
+	void resumeThreadReq(threadLifetimeMgmtOpCbFn callback);
+
+	/**
+	 * JOLTs this thread to begin processing after global initialization.
+	 *
+	 * JOLTing is the mechanism that allows threads to enter their main
+	 * event loops and set up TLS vars after all global constructors have
+	 * completed. This prevents race conditions during system startup.
+	 */
+	void joltThreadReq(threadLifetimeMgmtOpCbFn callback);
+
+	// CPU management methods
+	void pinToCpu(int cpuId);
+
+public:
+	int pinnedCpuId;
+	boost::asio::io_service pause_io_service;
+	boost::asio::io_service::work pause_work;
+	Mind& parent;
+	std::thread thread;
+
+public:
 	class ThreadLifetimeMgmtOp;
 	class MindShutdownIndOp;
 };
 
 namespace mrntt {
-extern std::shared_ptr<ComponentThread> mrntt;
+extern std::shared_ptr<MarionetteThread> mrntt;
 }
 
 } // namespace smo

smocore/include/marionette/marionette.h

@@ -3,16 +3,17 @@
 
 #include <cstdint>
 #include <atomic>
+#include <memory>
+#include <componentThread.h>
 
 namespace smo {
+
+class MarionetteThread;
+
 namespace mrntt {
 
 extern std::atomic<int> exitCode;
 
-class Marionette
-{
-};
-
 void exitMarionetteLoop();
 
 } // namespace mrntt

smocore/include/mind.h

@@ -27,12 +27,12 @@ public:
 	void finalizeBodyReq(mindLifetimeMgmtOpCbFn callback);
 
 	// ComponentThread access methods
-	std::shared_ptr<ComponentThread> getComponentThread(
+	std::shared_ptr<MindThread> getComponentThread(
 		ComponentThread::ThreadId id) const;
-	std::shared_ptr<ComponentThread> getComponentThread(
+	std::shared_ptr<MindThread> getComponentThread(
 		const std::string& name) const;
 	// Get all this Mind's component threads.
-	std::vector<std::shared_ptr<ComponentThread>> getMindThreads() const;
+	std::vector<std::shared_ptr<MindThread>> getMindThreads() const;
 
 	// Thread management methods (moved from ComponentThread)
 	typedef std::function<void()> mindThreadLifetimeMgmtOpCbFn;
@@ -76,7 +76,7 @@ private:
 	bool threadsHaveBeenJolted = false,
 		bodyComponentInitialized = false;
 	// Collection of ComponentThread instances (excluding marionette)
-	std::vector<std::shared_ptr<ComponentThread>> componentThreads;
+	std::vector<std::shared_ptr<MindThread>> componentThreads;
 
 	class MindLifetimeMgmtOp;
 	class MindThreadLifetimeMgmtOp;

smocore/marionette/marionette.cpp

@@ -13,13 +13,6 @@
 
 namespace smo {
 
-// Global Mind instance
-std::shared_ptr<Mind> globalMind = std::make_shared<Mind>();
-
-// Global marionette thread instance
-std::shared_ptr<ComponentThread> mrntt::mrntt =
-	std::make_shared<ComponentThread>(ComponentThread::MRNTT, *globalMind);
-
 CrtCommandLineArgs crtCommandLineArgs(0, nullptr, nullptr);
 
 void CrtCommandLineArgs::set(int argc, char *argv[], char *envp[])
@@ -27,8 +20,13 @@ void CrtCommandLineArgs::set(int argc, char *argv[], char *envp[])
     crtCommandLineArgs = CrtCommandLineArgs(argc, argv, envp);
 }
 
+// Global Mind instance
+std::shared_ptr<Mind> globalMind;
+
 namespace mrntt {
 std::atomic<int> exitCode;
+// Global marionette thread instance
+std::shared_ptr<MarionetteThread> mrntt = std::make_shared<MarionetteThread>();
 
 void exitMarionetteLoop()
 {
@@ -39,7 +37,7 @@ void exitMarionetteLoop()
 
 } // namespace mrntt
 
-void ComponentThread::marionetteMain(ComponentThread& self)
+void MarionetteThread::main(MarionetteThread& self)
 {
     // Wait for CRT's main() to post us the command line args.
     std::cout << __func__ << ": Waiting for command line JOLT" << std::endl;
@@ -111,6 +109,8 @@ void ComponentThread::marionetteMain(ComponentThread& self)
 			 * The latter is cleaner and more resource-respecting. The former is
 			 * easier to implement.
 			 */
+			// Create new Mind instance just before initializeReq
+			globalMind = std::make_shared<Mind>();
 			globalMind->initializeReq(
 			[](bool success)
 			{

smocore/mind.cpp

@@ -10,20 +10,25 @@ namespace smo {
 
 Mind::Mind(void)
 	: componentThreads{
-		std::make_shared<ComponentThread>(ComponentThread::DIRECTOR, *this),
-		std::make_shared<ComponentThread>(ComponentThread::SIMULATOR, *this),
-		std::make_shared<ComponentThread>(ComponentThread::SUBCONSCIOUS, *this),
-		std::make_shared<ComponentThread>(ComponentThread::BODY, *this),
-		std::make_shared<ComponentThread>(ComponentThread::WORLD, *this)
+		std::make_shared<MindThread>(ComponentThread::DIRECTOR, *this),
+		std::make_shared<MindThread>(ComponentThread::SIMULATOR, *this),
+		std::make_shared<MindThread>(ComponentThread::SUBCONSCIOUS, *this),
+		std::make_shared<MindThread>(ComponentThread::BODY, *this),
+		std::make_shared<MindThread>(ComponentThread::WORLD, *this)
 	}
 {
 }
 
-std::shared_ptr<ComponentThread>
+std::shared_ptr<MindThread>
 Mind::getComponentThread(ComponentThread::ThreadId id) const
 {
-    // Access the global marionette thread using ComponentThread::getMrntt()
-    if (id == ComponentThread::MRNTT) { return ComponentThread::getMrntt(); }
+	if (id == ComponentThread::MRNTT)
+	{
+		throw std::runtime_error(
+			std::string(__func__) +
+			": MRNTT is not a MindThread and cannot be returned by "
+			"getComponentThread");
+	}
 
     // Search through the vector for the thread with matching id
     for (auto& thread : componentThreads) {
@@ -32,14 +37,20 @@ Mind::getComponentThread(ComponentThread::ThreadId id) const
 
     // Throw exception if no thread found
     throw std::runtime_error(std::string(__func__) +
-        ": No ComponentThread found with ID "
+        ": No MindThread found with ID "
 		+ std::to_string(static_cast<int>(id)));
 }
 
-std::shared_ptr<ComponentThread>
+std::shared_ptr<MindThread>
 Mind::getComponentThread(const std::string& name) const
 {
-	if (name == "mrntt") { return ComponentThread::getMrntt(); }
+	if (name == "mrntt")
+	{
+		throw std::runtime_error(
+			std::string(__func__) +
+			": MRNTT is not a MindThread and cannot be returned by "
+			"getComponentThread");
+	}
 
     for (auto& thread : componentThreads) {
         if (thread->name == name) { return thread; }
@@ -47,10 +58,10 @@ Mind::getComponentThread(const std::string& name) const
 
     // Throw exception if no thread found
     throw std::runtime_error(std::string(__func__) +
-        ": No ComponentThread found with name '" + name + "'");
+        ": No MindThread found with name '" + name + "'");
 }
 
-std::vector<std::shared_ptr<ComponentThread>>
+std::vector<std::shared_ptr<MindThread>>
 Mind::getMindThreads() const
 {
     return componentThreads;