diff --git a/common/include/villas/kernel/devices/pci_device.hpp b/common/include/villas/kernel/devices/pci_device.hpp index 65794644b..8f6a10cf5 100644 --- a/common/include/villas/kernel/devices/pci_device.hpp +++ b/common/include/villas/kernel/devices/pci_device.hpp @@ -15,6 +15,8 @@ #include +#include + namespace villas { namespace kernel { namespace devices { @@ -58,7 +60,11 @@ struct Region { unsigned long long flags; }; -class PciDevice { +class PciDevice : public Device { +private: + static constexpr char PROBE_DEFAULT[] = "/sys/bus/pci/drivers_probe"; + static constexpr char OVERRIDE_DEFAULT[] = "driver_override"; + public: PciDevice(Id i, Slot s) : id(i), slot(s), log(Log::get("kernel:pci")) {} @@ -68,15 +74,17 @@ class PciDevice { bool operator==(const PciDevice &other); - // Get currently loaded driver for device - std::string getDriver() const; + // Implement device interface + std::optional> driver() const override; + std::optional iommu_group() const override; + std::string name() const override; + std::filesystem::path override_path() const override; + std::filesystem::path path() const override; + void probe() const override; // Bind a new LKM to the PCI device bool attachDriver(const std::string &driver) const; - // Return the IOMMU group of this PCI device or -1 if the device is not in a group - int getIommuGroup() const; - std::list getRegions() const; // Write 32-bit BAR value from to the PCI configuration space diff --git a/common/lib/kernel/devices/pci_device.cpp b/common/lib/kernel/devices/pci_device.cpp index ef8a8b5d0..2324c6013 100644 --- a/common/lib/kernel/devices/pci_device.cpp +++ b/common/lib/kernel/devices/pci_device.cpp @@ -18,10 +18,12 @@ #include #include +#include #include #include using namespace villas::kernel::devices; +using villas::utils::write_to_file; #define PCI_BASE_ADDRESS_N(n) (PCI_BASE_ADDRESS_0 + sizeof(uint32_t) * (n)) @@ -311,8 +313,7 @@ std::list PciDevice::getRegions() const { return regions; } - -std::string PciDevice::getDriver() const { +std::optional> PciDevice::driver() const { int ret; char sysfs[1024], syml[1024]; memset(syml, 0, sizeof(syml)); @@ -323,13 +324,15 @@ std::string PciDevice::getDriver() const { struct stat st; ret = stat(sysfs, &st); if (ret) - return ""; + return std::nullopt; ret = readlink(sysfs, syml, sizeof(syml)); if (ret < 0) throw SystemError("Failed to follow link: {}", sysfs); - return basename(syml); + auto driver = std::make_optional(std::make_unique( + "/sys/bus/pci/drivers/" + std::string(basename(syml)))); + return driver; } bool PciDevice::attachDriver(const std::string &driver) const { @@ -423,7 +426,7 @@ void PciDevice::writeBar(uint32_t addr, unsigned barNum) { file.write(reinterpret_cast(&addr), sizeof(addr)); } -int PciDevice::getIommuGroup() const { +std::optional PciDevice::iommu_group() const { int ret; char *group; @@ -437,11 +440,11 @@ int PciDevice::getIommuGroup() const { ret = readlink(sysfs, link, sizeof(link)); if (ret < 0) - return -1; + return std::nullopt; group = basename(link); - return atoi(group); + return std::make_optional(atoi(group)); } std::fstream PciDevice::openSysFs(const std::string &subPath, @@ -457,3 +460,33 @@ std::fstream PciDevice::openSysFs(const std::string &subPath, return file; } + +// TODO: test +std::string PciDevice::name() const { + char sysfs[1024]; + + snprintf(sysfs, sizeof(sysfs), "%04x:%02x:%02x.%x", slot.domain, slot.bus, + slot.device, slot.function); + + return std::string(sysfs); +} + +// TODO: test +std::filesystem::path PciDevice::path() const { + char sysfs[1024]; + + snprintf(sysfs, sizeof(sysfs), "%04x:%02x:%02x.%x", slot.domain, slot.bus, + slot.device, slot.function); + + return sysfs; +} + +// TODO: test +std::filesystem::path PciDevice::override_path() const { + + return this->path() / OVERRIDE_DEFAULT; +} + +void PciDevice::probe() const { + write_to_file(this->name(), this->PROBE_DEFAULT); +} diff --git a/common/lib/kernel/vfio_container.cpp b/common/lib/kernel/vfio_container.cpp index 76923aa08..595edf9fd 100644 --- a/common/lib/kernel/vfio_container.cpp +++ b/common/lib/kernel/vfio_container.cpp @@ -197,7 +197,7 @@ std::shared_ptr Container::attachDevice(devices::PciDevice &pdev) { throw RuntimeError("Failed to load kernel driver: vfio_pci"); // Bind PCI card to vfio-pci driver if not already bound - if (pdev.getDriver() != kernelDriver) { + if ( !(pdev.driver().has_value()) || pdev.driver().value()->name() != kernelDriver) { log->debug("Bind PCI card to kernel driver '{}'", kernelDriver); pdev.attachDriver(kernelDriver); } @@ -215,7 +215,7 @@ std::shared_ptr Container::attachDevice(devices::PciDevice &pdev) { } // Get IOMMU group of device - int index = isIommuEnabled() ? pdev.getIommuGroup() : 0; + int index = isIommuEnabled() ? pdev.iommu_group().value() : 0; if (index < 0) { ret = kernel::getCmdlineParam("intel_iommu", iommu_state, sizeof(iommu_state)); diff --git a/fpga/include/villas/fpga/ips/intc.hpp b/fpga/include/villas/fpga/ips/intc.hpp index a682582d0..af852af5b 100644 --- a/fpga/include/villas/fpga/ips/intc.hpp +++ b/fpga/include/villas/fpga/ips/intc.hpp @@ -27,7 +27,7 @@ class InterruptController : public Core { virtual bool init() override; virtual bool stop() override; - bool enableInterrupt(IrqMaskType mask, bool polling); + virtual bool enableInterrupt(IrqMaskType mask, bool polling); bool enableInterrupt(IrqPort irq, bool polling) { return enableInterrupt(1 << irq.num, polling); } @@ -38,7 +38,7 @@ class InterruptController : public Core { ssize_t waitForInterrupt(int irq); ssize_t waitForInterrupt(IrqPort irq) { return waitForInterrupt(irq.num); } -private: +protected: static constexpr char registerMemory[] = "reg0"; std::shared_ptr vfioDevice = nullptr; diff --git a/fpga/include/villas/fpga/ips/platform_intc.hpp b/fpga/include/villas/fpga/ips/platform_intc.hpp new file mode 100644 index 000000000..e90989ddb --- /dev/null +++ b/fpga/include/villas/fpga/ips/platform_intc.hpp @@ -0,0 +1,68 @@ +/* Platform based card + * + * Author: Pascal Bauer + * + * SPDX-FileCopyrightText: 2024-2025 Pascal Bauer + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include +#include +#include + +namespace villas { +namespace fpga { +namespace ip { + +class PlatformInterruptController + : public villas::fpga::ip::InterruptController { +public: + std::shared_ptr core; + + PlatformInterruptController(std::shared_ptr core) + : core(core) { + logger = Log::get("PlatformIntc"); + this->id = IpIdentifier("acs:software:platformIntc:1.0", + "PlatformInterruptcontroller"); + }; + + std::list getMemoryBlocks() const override { return {}; } + + bool init() override { + auto platform_card = dynamic_cast(card); + for (auto core_connection : platform_card->core_connections) { + if (core_connection.ip->getBaseaddr() == core->getBaseaddr()) { + vfioDevice = core_connection.device_connection.vfio_device; + logger->info("Enabled VFIO-Interrupt for {} with addr {:#x} ", + core->getInstanceName(), core->getBaseaddr()); + } + } + + if (vfioDevice == nullptr) { + logger->warn("Could not find Vfio device for ip: {}", + core->getInstanceName()); + return -1; + } + + irq_vectors = vfioDevice->initEventFds(); + + return true; + }; + + bool enableInterrupt(IrqMaskType mask, bool polling) override { + logger->debug("Enabling interrupt (platform)"); + for (size_t i = 0; i < irq_vectors[0].numFds; i++) { + if (mask & (1 << i)) + this->polling[i] = polling; + } + return true; + }; +}; + +} // namespace ip +} // namespace fpga +} // namespace villas diff --git a/fpga/lib/core.cpp b/fpga/lib/core.cpp index 691426057..477b6abbb 100644 --- a/fpga/lib/core.cpp +++ b/fpga/lib/core.cpp @@ -21,7 +21,9 @@ #include #include #include +#include #include +#include using namespace villas::fpga; using namespace villas::fpga::ip; @@ -30,6 +32,7 @@ using namespace villas::fpga::ip; // same order as they appear in this list, i.e. first here will be initialized // first. static std::list vlnvInitializationOrder = { + Vlnv("xilinx.com:ip:zynq_ultra_ps_e:"), Vlnv("xilinx.com:ip:axi_pcie:"), Vlnv("xilinx.com:ip:xdma:"), Vlnv("xilinx.com:module_ref:axi_pcie_intc:"), @@ -137,7 +140,7 @@ CoreFactory::configureIps(std::list orderedIps, json_t *json_ips, // If something goes wrong with initialization, the shared_ptr will // take care to desctruct the Core again as it is not pushed to // the list and will run out of scope. - auto ip = std::unique_ptr(f->make()); + auto ip = std::shared_ptr(f->make()); if (ip == nullptr) { logger->warn("Cannot create an instance of {}", f->getName()); @@ -182,15 +185,24 @@ CoreFactory::configureIps(std::list orderedIps, json_t *json_ips, for (auto &configuredIp : configuredIps) { if (*configuredIp == irqControllerName) { + + //! Assuming there is one parsed ip as intc, which can be reused from the heap intc = dynamic_cast(configuredIp.get()); + break; } } if (intc == nullptr) { - logger->error("Interrupt Controller {} for IRQ {} not found", - irqControllerName, irqName); - continue; + if (irqControllerName.find("zynq") != std::string::npos) { + intc = new PlatformInterruptController(ip); + intc->card = card; + configuredIps.push_back(std::shared_ptr(intc)); + } else { + logger->error("Interrupt Controller {} for IRQ {} not found", + irqControllerName, irqName); + continue; + } } int num; @@ -201,7 +213,7 @@ CoreFactory::configureIps(std::list orderedIps, json_t *json_ips, continue; } logger->debug("IRQ: {} -> {}:{}", irqName, irqControllerName, num); - ip->irqs[irqName] = {num, intc, ""}; + ip->irqs[irqName] = {0, intc, ""}; } } @@ -344,6 +356,14 @@ std::list> CoreFactory::make(Card *card, std::list> configuredIps = configureIps(orderedIps, json_ips, card); // Successfully configured IPs + /* If Platform then connect the ips via vfio + The the ordering is important because initIps() assumes memory of ip is accessible. + */ + auto platform_card = dynamic_cast(card); + if (platform_card != nullptr) { + platform_card->connectVFIOtoIps(configuredIps); + } + initIps(configuredIps, card); return card->ips; diff --git a/fpga/lib/utils.cpp b/fpga/lib/utils.cpp index 1449c9b96..575c99c72 100644 --- a/fpga/lib/utils.cpp +++ b/fpga/lib/utils.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -246,19 +247,20 @@ fpga::createCard(json_t *config, const std::filesystem::path &searchPath, throw ConfigError(config, err, "interface", "Failed to parse interface name for card {}", card_name); } + std::string interfaceNameStr(interfaceName); + std::shared_ptr card = nullptr; if (interfaceNameStr == "pcie") { - auto card = fpga::PCIeCardFactory::make(config, std::string(card_name), - vfioContainer, searchPath); - if (card) { - return card; - } - return nullptr; + card = fpga::PCIeCardFactory::make(config, std::string(card_name), + vfioContainer, searchPath); } else if (interfaceNameStr == "platform") { - throw RuntimeError("Platform interface not implemented yet"); + card = fpga::PlatformCardFactory::make(config, std::string(card_name), + vfioContainer, searchPath); } else { throw RuntimeError("Unknown interface type {}", interfaceNameStr); } + + return card; } int fpga::createCards(json_t *config, diff --git a/miob copy.conf b/miob copy.conf new file mode 100644 index 000000000..a27454fb4 --- /dev/null +++ b/miob copy.conf @@ -0,0 +1,68 @@ +logging = { + level = "info" +} + +fpgas = { + zcu106 = { + interface = "platform", + ips = "/home/root/codebase/node/etc/fpga/zcu106_aurora_dino/zcu106_aurora_dino_240808-2.json", # Absolute path for remote debugging + #ignore_ips = ["dino_dinoif_dac_0", "dino_dinoif_fast_nologic_0", "dino_registerif_0", "axi_iic_0"], + polling = true, + } +} + +nodes = { + fpga_0 = { + type = "fpga", + card = "zcu106" + # connect = ["loopback"] + connect = ["dino<->dma"] + #builtin = false, + lowLatencyMode = false, + }, + + signal_gen_0 = { + type = "signal.v2", + realtime = true, + rate = 200000, + in = { + signals = ( + { + name = "sine.v2", + signal = "sine", + frequency = 50, + amplitude = 10, + } + ) + }, + } +} + +paths = ( + { + builtin = false, + in = "signal_gen_0" + out = "fpga_0" + #hooks = ( + # { + # #type = "print" + # #output = "print_output_file.log" + # #format = "villas.human" + # }) + + }, + { + in = "fpga_0" + builtin = false, + #hooks = ( + #{ + #type = "print" + #output = "print_output_file.log" + #format = "villas.human" + #} + #) + } +) + + + diff --git a/miob.conf b/miob.conf new file mode 100644 index 000000000..37ec87f23 --- /dev/null +++ b/miob.conf @@ -0,0 +1,58 @@ +logging = { + level = "info" +} + +fpgas = { + zcu106 = { + interface = "platform", + ips = "/home/root/codebase/node/etc/fpga/zcu106-dino/zcu106-dino.json", # Absolute path for remote debugging + #ignore_ips = ["dino_dinoif_dac_0", "dino_dinoif_fast_nologic_0", "dino_registerif_0", "axi_iic_0"], + polling = true, + } +} + +nodes = { + fpga_0 = { + type = "fpga", + card = "zcu106" + connect = ["0<->dma"] + lowLatencyMode = false, + }, + + signal_gen_0 = { + type = "signal.v2", + realtime = true, + rate = 2, + in = { + signals = ( + { + name = "sine.v2", + signal = "sine", + frequency = 0.1, + amplitude = 10, + } + ) + }, + } +} + +paths = ( + { + in = "signal_gen_0" + out = "fpga_0" + hooks = ( + { + type = "print" + }) + + }, + { + in = "fpga_0" + hooks = ( + { + type = "print" + } + ) + } +) +