sim: Integrate simulator into UHD

This commit adds a device::register_device which allows uhd to start up
a simulator when uhd is called with the arguments type=sim. Creating the
device object creates a subprocess using pybind and an embedded
interpreter, and destroying the object cleans up those subprocesses.

Signed-off-by: Samuel O'Brien <sam.obrien@ni.com>

host/lib/CMakeLists.txt

@@ -69,6 +69,7 @@ LIBUHD_REGISTER_COMPONENT("USRP1" ENABLE_USRP1 ON "ENABLE_LIBUHD;ENABLE_USB" OFF
 LIBUHD_REGISTER_COMPONENT("USRP2" ENABLE_USRP2 ON "ENABLE_LIBUHD" OFF OFF)
 LIBUHD_REGISTER_COMPONENT("X300" ENABLE_X300 ON "ENABLE_LIBUHD" OFF OFF)
 LIBUHD_REGISTER_COMPONENT("MPMD" ENABLE_MPMD ON "ENABLE_LIBUHD" OFF OFF)
+LIBUHD_REGISTER_COMPONENT("SIM" ENABLE_SIM ON "ENABLE_LIBUHD;ENABLE_MPMD;ENABLE_PYTHON_API" OFF OFF)
 LIBUHD_REGISTER_COMPONENT("N300" ENABLE_N300 ON "ENABLE_LIBUHD;ENABLE_MPMD" OFF OFF)
 LIBUHD_REGISTER_COMPONENT("N320" ENABLE_N320 ON "ENABLE_LIBUHD;ENABLE_MPMD" OFF OFF)
 LIBUHD_REGISTER_COMPONENT("E320" ENABLE_E320 ON "ENABLE_LIBUHD;ENABLE_MPMD" OFF OFF)
@@ -187,6 +188,26 @@ if(DEFINED LIBUHD_OUTPUT_NAME)
     set_target_properties(uhd PROPERTIES OUTPUT_NAME ${LIBUHD_OUTPUT_NAME})
 endif(DEFINED LIBUHD_OUTPUT_NAME)
 
+if(ENABLE_SIM)
+    # Get python include dirs
+    include_directories(${PYTHON_INCLUDE_DIRS})
+    set(PYBIND11_INCLUDE_DIR
+        "${CMAKE_SOURCE_DIR}/lib/deps/pybind11/include"
+        CACHE
+        STRING
+        "Location of PyBind11 includes"
+    )
+    include_directories(${PYBIND11_INCLUDE_DIR})
+
+    # For PYUHD we don't link against the python libraries, but when calling
+    # python instead of being called by it, we have to.
+    target_link_libraries(uhd ${PYTHON_LIBRARIES})
+
+    if(APPLE)
+        target_link_options(pyuhd PRIVATE "LINKER:-undefined,dynamic_lookup")
+    endif(APPLE)
+endif(ENABLE_SIM)
+
 if(NOT UHDHOST_PKG) #Syntax makes it unusable by UHD_INSTALL
     install(TARGETS uhd
         LIBRARY DESTINATION ${LIBRARY_DIR} COMPONENT libraries # .so file

host/lib/usrp/mpmd/CMakeLists.txt

@@ -18,6 +18,12 @@ if(ENABLE_MPMD)
         ${CMAKE_CURRENT_SOURCE_DIR}/mpmd_link_if_ctrl_udp.cpp
     )
 
+    if(ENABLE_SIM)
+        LIBUHD_APPEND_SOURCES(
+            ${CMAKE_CURRENT_SOURCE_DIR}/sim_find.cpp
+        )
+    endif(ENABLE_SIM)
+
     if(ENABLE_DPDK)
         include_directories(${DPDK_INCLUDE_DIRS})
         set_property(

host/lib/usrp/mpmd/mpmd_impl.cpp

@@ -152,11 +152,10 @@ mpmd_impl::mpmd_impl(const device_addr_t& device_args)
 {
     const device_addrs_t mb_args_without_prefs = separate_device_addr(device_args);
     device_addrs_t mb_args;
-    for (size_t i = 0; i < mb_args_without_prefs.size(); ++i)
-    {
+    for (size_t i = 0; i < mb_args_without_prefs.size(); ++i) {
         mb_args.push_back(prefs::get_usrp_args(mb_args_without_prefs[i]));
     }
-    const size_t num_mboards     = mb_args.size();
+    const size_t num_mboards = mb_args.size();
     _mb.reserve(num_mboards);
     const bool serialize_init = device_args.has_key("serialize_init");
     const bool skip_init      = device_args.has_key("skip_init");
@@ -208,6 +207,14 @@ mpmd_impl::mpmd_impl(const device_addr_t& device_args)
 }
 
 mpmd_impl::~mpmd_impl()
+{
+    _deinit();
+}
+
+/*****************************************************************************
+ * Protected methods
+ ****************************************************************************/
+void mpmd_impl::_deinit()
 {
     _tree.reset();
     _mb.clear();

host/lib/usrp/mpmd/mpmd_impl.hpp

@@ -227,6 +227,10 @@ public:
         return _mb.at(mb_idx)->get_mb_iface();
     }
 
+protected:
+    //! Destroys the mboard_impls and the device_tree
+    void _deinit();
+
 private:
     /*************************************************************************
      * Private methods/helpers

host/lib/usrp/mpmd/sim_find.cpp

@@ -0,0 +1,177 @@
+//
+// Copyright 2020 Ettus Research, a National Instruments Brand
+//
+// SPDX-License-Identifier: GPL-3.0-or-later
+//
+
+#include "mpmd_devices.hpp"
+#include "mpmd_impl.hpp"
+#include <uhd/device.hpp>
+#include <uhd/utils/static.hpp>
+#include <uhdlib/rfnoc/rfnoc_device.hpp>
+// Need this import because pybind doesn't have an equivalent to Py_IsInitialized()
+#include <Python.h>
+#include <pybind11/embed.h>
+#include <pybind11/pybind11.h>
+#include <chrono>
+#include <cstdint>
+#include <cstdio>
+#include <memory>
+
+using namespace uhd;
+using namespace uhd::mpmd;
+using namespace std::chrono_literals;
+namespace py = pybind11;
+
+constexpr auto SIMULATOR_EXIT_TIMEOUT    = 5s;
+constexpr auto SIMULATOR_STARTUP_TIMEOUT = 5s;
+
+// There can only be one python interpreter instantiated at a time
+// The guard means we only destroy the interpreter if we created it
+static std::unique_ptr<py::scoped_interpreter> interpreter_guard;
+
+void ensure_python_interpreter()
+{
+    // This call is needed because the interpreter may already be running
+    // i.e. UHD is being called from python through pyuhd
+    if (not Py_IsInitialized()) {
+        interpreter_guard = std::make_unique<py::scoped_interpreter>();
+    }
+}
+
+py::object get_simulator_module()
+{
+    try {
+        return py::module::import("usrp_mpm.process_manager");
+    } catch (const py::error_already_set& ex) {
+        std::string message("Simulator failed to import: ");
+        message.append(ex.what());
+        message.append("\nPYTHONPATH: ");
+        auto pythonpath =
+            py::str(py::module::import("sys").attr("path")).cast<std::string>();
+        message.append(pythonpath);
+        throw std::runtime_error(message);
+    }
+}
+
+device_addrs_t mpmd_find_with_addr(
+    const std::string& mgmt_addr, const device_addr_t& hint_);
+
+void shutdown_process(py::object& process_manager)
+{
+    // TODO: Sometimes during a TX, the simulator gets shutdown before all of the packets
+    // are sent
+    py::object stop_fn = process_manager.attr("stop");
+    const double timeout_floating =
+        std::chrono::duration<double>(SIMULATOR_EXIT_TIMEOUT).count();
+    const bool result = stop_fn(timeout_floating).cast<bool>();
+    if (!result) {
+        UHD_LOG_WARNING("SIM",
+            "Simulator Subprocess did not exit, manual cleanup of subprocesses may "
+            "be necessary.")
+        process_manager.attr("terminate")();
+    }
+}
+
+class sim_impl : public mpmd_impl
+{
+public:
+    sim_impl(const uhd::device_addr_t& device_addr, py::object process_manager)
+        : mpmd_impl(device_addr), _process_manager(std::move(process_manager))
+    {
+    }
+
+    ~sim_impl()
+    {
+        // Destroys the mb_ifaces, causing mpm to be unclaimed before shutting down the
+        // simulator
+        _deinit();
+        shutdown_process(_process_manager);
+    }
+
+private:
+    // This is an object of type ProcessManager
+    // See mpm/python/usrp_mpm/process_manager.py
+    py::object _process_manager;
+};
+
+device_addrs_t sim_find(const device_addr_t& hint_)
+{
+    device_addrs_t simulators;
+    if (hint_.has_key("type") && hint_["type"] == "sim") {
+        simulators.push_back(hint_);
+        // Set addr to localhost
+        simulators.back()["addr"]      = "127.0.0.1";
+        simulators.back()["mgmt_addr"] = "127.0.0.1";
+        // So discovery doesn't complain about hint mismatch
+        simulators.back()["type"] = MPM_CATCHALL_DEVICE_TYPE;
+    }
+    return simulators;
+}
+
+/*! Ensure that the simulator is loaded by pinging the discovery port until it responds or
+ *  the function times out
+ */
+bool check_simulator_status(
+    const device_addr_t& device_addr, std::chrono::milliseconds timeout)
+{
+    const auto timeout_time = std::chrono::steady_clock::now() + timeout;
+    while (std::chrono::steady_clock::now() < timeout_time) {
+        const auto devices = mpmd_find_with_addr(device_addr["mgmt_addr"], device_addr);
+        if (!devices.empty()) {
+            return true;
+        }
+        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+    }
+    return false;
+}
+
+device::sptr sim_make(const device_addr_t& device_args)
+{
+    // Ensure the interpreter is loaded
+    ensure_python_interpreter();
+    py::object manager_module = get_simulator_module();
+    py::object manager_class  = manager_module.attr("ProcessManager");
+
+    std::string config_arg("--default-args=config=");
+    if (not device_args.has_key("config")) {
+        throw std::runtime_error(
+            "Please specify a config file using the args key 'config'");
+    }
+    config_arg.append(device_args["config"]);
+
+    py::list process_args;
+    process_args.append(py::str(config_arg));
+
+    if (device_args.has_key("log_level")) {
+        std::string level = device_args["log_level"];
+        if (level == "trace") {
+            process_args.append(py::str("-vv"));
+        } else if (level == "debug") {
+            process_args.append(py::str("-v"));
+        } else if (level == "info") {
+            // No-op
+        } else if (level == "warning") {
+            process_args.append(py::str("-q"));
+        } else if (level == "error") {
+            process_args.append(py::str("-qq"));
+        }
+    }
+
+    py::object process_manager = manager_class(process_args);
+    process_manager.attr("start")();
+
+    const uint32_t pid = process_manager.attr("pid")().cast<uint32_t>();
+    UHD_LOG_INFO("SIM", "Starting simulator as pid " << pid);
+    if (not check_simulator_status(device_args, SIMULATOR_STARTUP_TIMEOUT)) {
+        shutdown_process(process_manager);
+        throw std::runtime_error("Simulator Startup timed out!");
+    }
+    return static_cast<device::sptr>(
+        std::make_shared<sim_impl>(device_args, std::move(process_manager)));
+}
+
+UHD_STATIC_BLOCK(register_sim_device)
+{
+    device::register_device(&sim_find, &sim_make, device::USRP);
+}

mpm/python/usrp_hwd.py

@@ -24,12 +24,22 @@ from threading import Event, Thread
 
 _PROCESSES = []
 _KILL_EVENT = Event()
+# This Global Variable is used by the Simulator to make the spawn_processes,
+# and by extension the main method, exit without waiting for the simulator to stop.
+# See process_manager.py:bootstrap() for more information.
+JOIN_PROCESSES = True
 
 def setup_arg_parser():
     """
     Create an arg parser
     """
     parser = argparse.ArgumentParser(description="USRP Hardware Daemon")
+    parser.add_argument(
+        '--no-logbuf',
+        dest='use_logbuf',
+        help="Do not send log messages to UHD",
+        action="store_false",
+    )
     parser.add_argument(
         '--daemon',
         help="Run as daemon",
@@ -169,8 +179,9 @@ def spawn_processes(log, args):
     Thread(target=kill_thread, daemon=False).start()
     signal.signal(signal.SIGTERM, kill_time)
     signal.signal(signal.SIGINT, kill_time)
-    for proc in _PROCESSES:
-        proc.join()
+    if JOIN_PROCESSES:
+        for proc in _PROCESSES:
+            proc.join()
     return True
 
 def main():
@@ -181,6 +192,7 @@ def main():
     """
     args = parse_args()
     log = mpm.get_main_logger(
+        use_logbuf=args.use_logbuf,
         log_default_delta=args.verbose-args.quiet
     ).getChild('main')
     version_string = mpm.__version__

mpm/python/usrp_mpm/CMakeLists.txt

@@ -23,6 +23,7 @@ set(USRP_MPM_TOP_FILES
     ${CMAKE_CURRENT_SOURCE_DIR}/mpmutils.py
     ${CMAKE_CURRENT_SOURCE_DIR}/prefs.py
     ${CMAKE_CURRENT_SOURCE_DIR}/rpc_server.py
+    ${CMAKE_CURRENT_SOURCE_DIR}/process_manager.py
 )
 list(APPEND USRP_MPM_FILES ${USRP_MPM_TOP_FILES})
 add_subdirectory(chips)

mpm/python/usrp_mpm/process_manager.py

@@ -0,0 +1,60 @@
+#
+# Copyright 2020 Ettus Research, a National Instruments Brand
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+"""
+This module is used as an interface between the sim_find.cpp discovery
+and mboard_iface in uhd and the usrp_hwd python file. It manages
+starting and stopping the simulator subprocess and configuring logging
+"""
+from multiprocessing import Process, Event
+import sys
+try:
+    # Location if installed from using make install
+    import usrp_hwd
+except ImportError:
+    # Location if installed from libpyuhd using setuptools
+    from usrp_mpm import usrp_hwd
+
+class ProcessManager:
+    """This object is used to manage a simulator process which is launched
+    from a python interpreter rather than from an os shell or using systemd
+    """
+    def __init__(self, args):
+        """args are the command line arguments received by the simulator"""
+        self.stop_event = Event()
+        self.process = Process(target=_bootstrap, args=[args, self.stop_event])
+
+    def start(self):
+        """Launch the simulator's process"""
+        self.process.start()
+
+    def stop(self, timeout):
+        """Attempt to stop the simulator cleanly. Returns True if successful"""
+        self.stop_event.set()
+        self.process.join(timeout)
+        return self.process.exitcode is not None
+
+    def terminate(self):
+        """Forcefully terminates the simulator"""
+        self.process.terminate()
+
+    def pid(self):
+        """Returns the PID of the simulator subprocess"""
+        return int(self.process.pid)
+
+def _bootstrap(args, stop_event):
+    # Set args for new process
+    #
+    # Disable UHD log forwarding to avoid
+    # duplicate messages
+    sys.argv = ["usrp_hwd.py"] + args + ["--no-logbuf"]
+    # tell main() not to block
+    usrp_hwd.JOIN_PROCESSES = False
+    # Start the discovery and RPC processes
+    usrp_hwd.main()
+    # Wait for signal from other process
+    stop_event.wait()
+    # Stop the discovery and RPC processes
+    usrp_hwd.kill_time(None, None)