xilinx_alveo_u280: Add more IOs and enable HBM2.

This commit is contained in:
Sergiu Mosanu 2021-07-28 14:58:47 +02:00 committed by Florent Kermarrec
parent 401568c54e
commit 99ff82c75a
2 changed files with 303 additions and 60 deletions

View file

@ -11,7 +11,7 @@ from litex.build.xilinx import XilinxPlatform, VivadoProgrammer
# IOs -----------------------------------------------------------------------------------------------
_io = [
# Clk / Rst
# 100MHz Clk / Rst
("sysclk", 0,
Subsignal("n", Pins("BJ44"), IOStandard("DIFF_SSTL12")),
Subsignal("p", Pins("BJ43"), IOStandard("DIFF_SSTL12")),
@ -28,7 +28,6 @@ _io = [
("gpio_led", 2, Pins("D31"), IOStandard("LVCMOS18")),
# Switches
("gpio_sw", 0, Pins("J30"), IOStandard("LVCMOS18")),
("gpio_sw", 1, Pins("J32"), IOStandard("LVCMOS18")),
("gpio_sw", 2, Pins("K32"), IOStandard("LVCMOS18")),
@ -132,6 +131,87 @@ _io = [
Subsignal("reset_n", Pins("BH12"), IOStandard("LVCMOS12")),
Misc("SLEW=FAST")
),
# I2C (not tested on hardware)
("i2c_rst_n", 0, Pins("B31"), IOStandard("LVCMOS18")),
("i2c", 0,
Subsignal("scl", Pins("C30"), IOStandard("LVCMOS18")),
Subsignal("sda", Pins("C33"), IOStandard("LVCMOS18")),
),
# QSFP Clock (not tested on hardware)
("qsfp_156mhz_clock", 0,
Subsignal("n", Pins("T43")),
Subsignal("p", Pins("T42")),
),
("qsfp_156mhz_clock", 1,
Subsignal("n", Pins("P43")),
Subsignal("p", Pins("P42")),
),
# PCIe (hardware tests in progress)
("pcie_x16", 0,
Subsignal("rst_n", Pins("BH26"), IOStandard("LVCMOS18")),
Subsignal("clk_n", Pins("AR14")),
Subsignal("clk_p", Pins("AR15")),
Subsignal("rx_n", Pins(
"AL1 AM3 AN5 AN1 AP3 AR1 AT3 AU1",
"AV3 AW5 AW1 AY3 BA5 BA1 BB3 BC1")),
Subsignal("rx_p", Pins(
"AL2 AM4 AN6 AN2 AP4 AR2 AT4 AU2",
"AV4 AW6 AW2 AY4 BA6 BA2 BB4 BC2")),
Subsignal("tx_n", Pins(
"AL10 AM8 AN10 AP8 AR10 AR6 AT8 AU10",
"AU6 AV8 AW10 AY8 BA10 BB8 BC10 BC6")),
Subsignal("tx_p", Pins(
"AL11 AM9 AN11 AP9 AR11 AR7 AT9 AU11",
"AU7 AV9 AW11 AY9 BA11 BB9 BC11 BC7")),
),
# PCIe (hardware tests in progress)
("pcie_x4", 0,
Subsignal("rst_n", Pins("BH26"), IOStandard("LVCMOS18")),
Subsignal("clk_n", Pins("AR14")),
Subsignal("clk_p", Pins("AR15")),
Subsignal("rx_n", Pins("AL1 AM3 AN5 AN1")),
Subsignal("rx_p", Pins("AL2 AM4 AN6 AN2")),
Subsignal("tx_n", Pins("AL10 AM8 AN10 AP8")),
Subsignal("tx_p", Pins("AL11 AM9 AN11 AP9")),
),
# QSFP28 (not tested on hardware)
("qsfp28", 0,
Subsignal("clk_n", Pins("R41")),
Subsignal("clk_p", Pins("R40")),
#Subsignal("fs0", Pins(""), IOStandard("LVCMOS18")), # not found in u280 pins
#Subsignal("fs1", Pins(""), IOStandard("LVCMOS18")), # not found in u280 pins
Subsignal("intl", Pins("B32")),
Subsignal("lpmode", Pins("C29")),
Subsignal("modprsl", Pins("A33")),
Subsignal("modskll", Pins("A31")),
#Subsignal("refclk_reset", Pins(""), IOStandard("LVCMOS12")), # not found in u280 pins
Subsignal("resetl", Pins("B30")),
Subsignal("rxn", Pins("L54 K52 J54 H52")),
Subsignal("rxp", Pins("L53 K51 J53 H51")),
Subsignal("txn", Pins("L49 L45 K47 J49")),
Subsignal("txp", Pins("L48 L44 K46 J48")),
),
("qsfp28", 1,
Subsignal("clk_n", Pins("M43")),
Subsignal("clk_p", Pins("M42")),
#Subsignal("fs0", Pins(""), IOStandard("LVCMOS18")), # not found in u280 pins
#Subsignal("fs1", Pins(""), IOStandard("LVCMOS18")), # not found in u280 pins
Subsignal("intl", Pins("E29")),
Subsignal("lpmode", Pins("F29")),
Subsignal("modprsl", Pins("F33")),
Subsignal("modskll", Pins("D30")),
#Subsignal("refclk_reset", Pins(""), IOStandard("LVCMOS12")), # not found in u280 pins
Subsignal("resetl", Pins("E33")),
Subsignal("rxn", Pins("G54 F52 E54 D52")),
Subsignal("rxp", Pins("G53 F51 E53 D51")),
Subsignal("txn", Pins("G49 E49 C49 A50")),
Subsignal("txp", Pins("G48 E48 C48 A49")),
),
]
# Connectors ---------------------------------------------------------------------------------------
@ -169,12 +249,15 @@ class Platform(XilinxPlatform):
self.add_platform_command("set_property INTERNAL_VREF 0.84 [get_iobanks 70]")
# Other suggested configurations
# self.add_platform_command("set_property CONFIG_VOLTAGE 1.8 [current_design]")
# self.add_platform_command("set_property BITSTREAM.CONFIG.CONFIGFALLBACK Enable [current_design]")
# self.add_platform_command("set_property CONFIG_MODE SPIx4 [current_design]")
# self.add_platform_command("set_property BITSTREAM.CONFIG.SPI_BUSWIDTH 4 [current_design]")
# self.add_platform_command("set_property BITSTREAM.CONFIG.CONFIGRATE 85.0 [current_design]")
# self.add_platform_command("set_property BITSTREAM.CONFIG.EXTMASTERCCLK_EN disable [current_design]")
# self.add_platform_command("set_property BITSTREAM.CONFIG.SPI_FALL_EDGE YES [current_design]")
# self.add_platform_command("set_property BITSTREAM.CONFIG.UNUSEDPIN Pullup [current_design]")
# self.add_platform_command("set_property BITSTREAM.CONFIG.SPI_32BIT_ADDR Yes [current_design]")
self.add_platform_command("set_property CONFIG_VOLTAGE 1.8 [current_design]")
self.add_platform_command("set_property BITSTREAM.CONFIG.CONFIGFALLBACK Enable [current_design]")
self.add_platform_command("set_property CONFIG_MODE SPIx4 [current_design]")
self.add_platform_command("set_property BITSTREAM.CONFIG.SPI_BUSWIDTH 4 [current_design]")
self.add_platform_command("set_property BITSTREAM.CONFIG.CONFIGRATE 85.0 [current_design]")
self.add_platform_command("set_property BITSTREAM.CONFIG.EXTMASTERCCLK_EN disable [current_design]")
self.add_platform_command("set_property BITSTREAM.CONFIG.SPI_FALL_EDGE YES [current_design]")
self.add_platform_command("set_property BITSTREAM.CONFIG.UNUSEDPIN Pullup [current_design]")
self.add_platform_command("set_property BITSTREAM.CONFIG.SPI_32BIT_ADDR Yes [current_design]")
# For HBM2 IP in Vivado 2019.2 (https://www.xilinx.com/support/answers/72607.html)
self.add_platform_command("connect_debug_port dbg_hub/clk [get_nets apb_clk]")

View file

@ -4,6 +4,8 @@
# This file is part of LiteX-Boards.
#
# Copyright (c) 2021 Sergiu Mosanu <sm7ed@virginia.edu>
# Copyright (c) 2020-2021 Florent Kermarrec <florent@enjoy-digital.fr>
# Copyright (c) 2020 Antmicro <www.antmicro.com>
#
# SPDX-License-Identifier: BSD-2-Clause
@ -16,7 +18,10 @@ from litex_boards.platforms import alveo_u280
from litex.soc.cores.clock import *
from litex.soc.integration.soc_core import *
from litex.soc.integration.soc import SoCRegion
from litex.soc.integration.builder import *
from litex.soc.interconnect.axi import *
from litex.soc.interconnect.csr import *
from litex.soc.cores.led import LedChaser
from litedram.modules import MTA18ASF2G72PZ
@ -25,40 +30,174 @@ from litedram.phy import usddrphy
from litepcie.phy.usppciephy import USPPCIEPHY
from litepcie.software import generate_litepcie_software
# CRG ----------------------------------------------------------------------------------------------
from litedram.common import *
from litedram.frontend.axi import *
class _CRG(Module):
def __init__(self, platform, sys_clk_freq, ddram_channel):
self.rst = Signal()
self.clock_domains.cd_sys = ClockDomain()
self.clock_domains.cd_sys4x = ClockDomain(reset_less=True)
self.clock_domains.cd_pll4x = ClockDomain(reset_less=True)
self.clock_domains.cd_idelay = ClockDomain()
from litescope import LiteScopeAnalyzer
# HBM IP
class HBMIP(Module, AutoCSR):
"""Xilinx Virtex US+ High Bandwidth Memory 2 IP wrapper"""
def __init__(self, platform, hbm_ip_name="hbm_0"):
self.platform = platform
self.hbm_name = hbm_ip_name
self.axi = []
self.apb = []
self.hbm_params = {}
self.init_done = CSRStatus()
# # #
self.submodules.pll = pll = USMMCM(speedgrade=-2)
self.comb += pll.reset.eq(self.rst)
pll.register_clkin(platform.request("sysclk", ddram_channel), 100e6)
pll.create_clkout(self.cd_pll4x, sys_clk_freq*4, buf=None, with_reset=False)
pll.create_clkout(self.cd_idelay, 600e6, with_reset=False)
platform.add_false_path_constraints(self.cd_sys.clk, pll.clkin) # Ignore sys_clk to pll.clkin path created by SoC's rst.
class Open(Signal): pass
self.specials += [
Instance("BUFGCE_DIV", name="main_bufgce_div",
p_BUFGCE_DIVIDE=4,
i_CE=1, i_I=self.cd_pll4x.clk, o_O=self.cd_sys.clk),
Instance("BUFGCE", name="main_bufgce",
i_CE=1, i_I=self.cd_pll4x.clk, o_O=self.cd_sys4x.clk),
]
# Clocks -----------------------------------------------------------------------------------
# Ref = 100 MHz (HBM: 900 (225-900) MHz), drives internal PLL (1 per stack).
for i in range(2):
self.hbm_params[f"i_HBM_REF_CLK_{i:1d}"] = ClockSignal("hbm_ref")
self.submodules.idelayctrl = USIDELAYCTRL(cd_ref=self.cd_idelay, cd_sys=self.cd_sys)
# APB: 100 (50-100) MHz
for i in range(2):
self.hbm_params[f"i_APB_{i:1d}_PCLK"] = ClockSignal("apb")
self.hbm_params[f"i_APB_{i:1d}_PRESET_N"] = ~ResetSignal("apb")
# AXI: 450 (225-450) MHz
for i in range(32):
self.hbm_params[f"i_AXI_{i:02d}_ACLK"] = ClockSignal("axi")
self.hbm_params[f"i_AXI_{i:02d}_ARESET_N"] = ~ResetSignal("apb")
# AXI --------------------------------------------------------------------------------------
for i in range(32):
axi = AXIInterface(data_width=256, address_width=33, id_width=6)
self.axi.append(axi)
# AW Channel.
self.hbm_params[f"i_AXI_{i :02d}_AWADDR"] = axi.aw.addr
self.hbm_params[f"i_AXI_{i :02d}_AWBURST"] = axi.aw.burst
self.hbm_params[f"i_AXI_{i :02d}_AWID"] = axi.aw.id
self.hbm_params[f"i_AXI_{i :02d}_AWLEN"] = axi.aw.len
self.hbm_params[f"i_AXI_{i :02d}_AWSIZE"] = axi.aw.size
self.hbm_params[f"i_AXI_{i :02d}_AWVALID"] = axi.aw.valid
self.hbm_params[f"o_AXI_{i :02d}_AWREADY"] = axi.aw.ready
# W Channel.
self.hbm_params[f"i_AXI_{i:02d}_WDATA"] = axi.w.data
self.hbm_params[f"i_AXI_{i:02d}_WLAST"] = axi.w.last
self.hbm_params[f"i_AXI_{i:02d}_WSTRB"] = axi.w.strb
self.hbm_params[f"i_AXI_{i:02d}_WDATA_PARITY"] = 0 # FIXME: Manage parity?
self.hbm_params[f"i_AXI_{i:02d}_WVALID"] = axi.w.valid
self.hbm_params[f"o_AXI_{i:02d}_WREADY"] = axi.w.ready
# B Channel.
self.hbm_params[f"o_AXI_{i:02d}_BID"] = axi.b.id
self.hbm_params[f"o_AXI_{i:02d}_BRESP"] = axi.b.resp
self.hbm_params[f"o_AXI_{i:02d}_BVALID"] = axi.b.valid
self.hbm_params[f"i_AXI_{i:02d}_BREADY"] = axi.b.ready
# AR Channel.
self.hbm_params[f"i_AXI_{i:02d}_ARADDR"] = axi.ar.addr
self.hbm_params[f"i_AXI_{i:02d}_ARBURST"] = axi.ar.burst
self.hbm_params[f"i_AXI_{i:02d}_ARID"] = axi.ar.id
self.hbm_params[f"i_AXI_{i:02d}_ARLEN"] = axi.ar.len
self.hbm_params[f"i_AXI_{i:02d}_ARSIZE"] = axi.ar.size
self.hbm_params[f"i_AXI_{i:02d}_ARVALID"] = axi.ar.valid
self.hbm_params[f"o_AXI_{i:02d}_ARREADY"] = axi.ar.ready
# R Channel.
self.hbm_params[f"o_AXI_{i:02d}_RDATA_PARITY"] = Open() # FIXME: Manage parity?
self.hbm_params[f"o_AXI_{i:02d}_RDATA"] = axi.r.data
self.hbm_params[f"o_AXI_{i:02d}_RID"] = axi.r.id
self.hbm_params[f"o_AXI_{i:02d}_RLAST"] = axi.r.last
self.hbm_params[f"o_AXI_{i:02d}_RRESP"] = axi.r.resp
self.hbm_params[f"o_AXI_{i:02d}_RVALID"] = axi.r.valid
self.hbm_params[f"i_AXI_{i:02d}_RREADY"] = axi.r.ready
# APB --------------------------------------------------------------------------------------
# FIXME: Connect to CSR or Wishbone.
apb_complete = Signal(2)
for i in range(2):
self.hbm_params[f"i_APB_{i:1d}_PWDATA"] = 0
self.hbm_params[f"i_APB_{i:1d}_PADDR"] = 0
self.hbm_params[f"i_APB_{i:1d}_PENABLE"] = 0
self.hbm_params[f"i_APB_{i:1d}_PSEL"] = 0
self.hbm_params[f"i_APB_{i:1d}_PWRITE"] = 0
self.hbm_params[f"o_APB_{i:1d}_PRDATA"] = Open()
self.hbm_params[f"o_APB_{i:1d}_PREADY"] = Open()
self.hbm_params[f"o_APB_{i:1d}_PSLVERR"] = Open()
self.hbm_params[f"o_apb_complete_{i:1d}"] = apb_complete[i]
self.comb += self.init_done.status.eq(apb_complete == 0b11)
# Temperature ------------------------------------------------------------------------------
for i in range(2):
self.hbm_params[f"o_DRAM_{i:1d}_STAT_CATTRIP"] = Open()
self.hbm_params[f"o_DRAM_{i:1d}_STAT_TEMP"] = Open()
def add_sources(self, platform):
this_dir = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
os.system("wget https://github.com/litex-hub/litex-boards/files/6893157/hbm_0.xci.txt")
os.makedirs("ip/hbm", exist_ok=True)
os.system("mv hbm_0.xci.txt ip/hbm/hbm_0.xci")
platform.add_ip(os.path.join(this_dir, "ip", "hbm", self.hbm_name + ".xci"))
def do_finalize(self):
self.add_sources(self.platform)
self.specials += Instance(self.hbm_name, **self.hbm_params)
# CRG ----------------------------------------------------------------------------------------------
class _CRG(Module):
def __init__(self, platform, sys_clk_freq, ddram_channel, with_hbm):
if with_hbm:
self.clock_domains.cd_sys = ClockDomain()
self.clock_domains.cd_hbm_ref = ClockDomain()
self.clock_domains.cd_apb = ClockDomain()
else: # ddr4
self.rst = Signal()
self.clock_domains.cd_sys = ClockDomain()
self.clock_domains.cd_sys4x = ClockDomain(reset_less=True)
self.clock_domains.cd_pll4x = ClockDomain(reset_less=True)
self.clock_domains.cd_idelay = ClockDomain()
# # #
if with_hbm:
self.submodules.pll = pll = USMMCM(speedgrade=-2)
pll.register_clkin(platform.request("sysclk", ddram_channel), 100e6)
pll.create_clkout(self.cd_sys, sys_clk_freq)
pll.create_clkout(self.cd_hbm_ref, 100e6)
pll.create_clkout(self.cd_apb, 100e6)
platform.add_false_path_constraints(self.cd_sys.clk, self.cd_apb.clk)
else: # ddr4
self.submodules.pll = pll = USMMCM(speedgrade=-2)
self.comb += pll.reset.eq(self.rst)
pll.register_clkin(platform.request("sysclk", ddram_channel), 100e6)
pll.create_clkout(self.cd_pll4x, sys_clk_freq*4, buf=None, with_reset=False)
pll.create_clkout(self.cd_idelay, 600e6) #, with_reset=False
platform.add_false_path_constraints(self.cd_sys.clk, pll.clkin) # Ignore sys_clk to pll.clkin path created by SoC's rst.
self.specials += [
Instance("BUFGCE_DIV", name="main_bufgce_div",
p_BUFGCE_DIVIDE=4,
i_CE=1, i_I=self.cd_pll4x.clk, o_O=self.cd_sys.clk),
Instance("BUFGCE", name="main_bufgce",
i_CE=1, i_I=self.cd_pll4x.clk, o_O=self.cd_sys4x.clk),
# AsyncResetSynchronizer(self.cd_idelay, ~pll.locked),
]
self.submodules.idelayctrl = USIDELAYCTRL(cd_ref=self.cd_idelay, cd_sys=self.cd_sys)
# BaseSoC ------------------------------------------------------------------------------------------
class BaseSoC(SoCCore):
def __init__(self, sys_clk_freq=int(150e6), ddram_channel=0, with_pcie=False, with_led=False, **kwargs):
def __init__(self, sys_clk_freq=int(150e6), ddram_channel=0, with_pcie=False, with_led_chaser=False, with_hbm=False, **kwargs):
platform = alveo_u280.Platform()
if with_hbm:
assert 225e6 <= sys_clk_freq <= 450e6
# SoCCore ----------------------------------------------------------------------------------
SoCCore.__init__(self, platform, sys_clk_freq,
@ -67,25 +206,39 @@ class BaseSoC(SoCCore):
**kwargs)
# CRG --------------------------------------------------------------------------------------
self.submodules.crg = _CRG(platform, sys_clk_freq, ddram_channel)
self.submodules.crg = _CRG(platform, sys_clk_freq, ddram_channel, with_hbm)
# DDR4 SDRAM -------------------------------------------------------------------------------
if not self.integrated_main_ram_size:
self.submodules.ddrphy = usddrphy.USPDDRPHY(platform.request("ddram", ddram_channel),
memtype = "DDR4",
cmd_latency = 1, # seems to work better with cmd_latency=1
sys_clk_freq = sys_clk_freq,
iodelay_clk_freq = 600e6,
is_rdimm = True)
self.add_sdram("sdram",
phy = self.ddrphy,
module = MTA18ASF2G72PZ(sys_clk_freq, "1:4"),
size = 0x40000000,
l2_cache_size = kwargs.get("l2_size", 8192)
)
if with_hbm:
# JTAGBone --------------------------------------------------------------------------------
#self.add_jtagbone(chain=2) # Chain 1 already used by HBM2 debug probes.
# Firmware RAM (To ease initial LiteDRAM calibration support) ------------------------------
self.add_ram("firmware_ram", 0x20000000, 0x8000)
# Add HBM Core.
self.submodules.hbm = hbm = ClockDomainsRenamer({"axi": "sys"})(HBMIP(platform))
# Connect four of the HBM's AXI interfaces to the main bus of the SoC.
for i in range(4):
axi_hbm = hbm.axi[i]
axi_lite_hbm = AXILiteInterface(data_width=256, address_width=33)
self.submodules += AXILite2AXI(axi_lite_hbm, axi_hbm)
self.bus.add_slave(f"hbm{i}", axi_lite_hbm, SoCRegion(origin=0x4000_0000 + 0x1000_0000*i, size=0x1000_0000)) # 256MB.
else:
# DDR4 SDRAM -------------------------------------------------------------------------------
if not self.integrated_main_ram_size:
self.submodules.ddrphy = usddrphy.USPDDRPHY(platform.request("ddram", ddram_channel),
memtype = "DDR4",
cmd_latency = 1, # seems to work better with cmd_latency=1
sys_clk_freq = sys_clk_freq,
iodelay_clk_freq = 600e6,
is_rdimm = True)
self.add_sdram("sdram",
phy = self.ddrphy,
module = MTA18ASF2G72PZ(sys_clk_freq, "1:4"),
size = 0x40000000,
l2_cache_size = kwargs.get("l2_size", 8192)
)
# Firmware RAM (To ease initial LiteDRAM calibration support) ------------------------------
self.add_ram("firmware_ram", 0x20000000, 0x8000)
# PCIe -------------------------------------------------------------------------------------
if with_pcie:
@ -95,7 +248,7 @@ class BaseSoC(SoCCore):
self.add_pcie(phy=self.pcie_phy, ndmas=1)
# Leds -------------------------------------------------------------------------------------
if with_led:
if with_led_chaser:
self.submodules.leds = LedChaser(
pads = platform.request_all("gpio_led"),
sys_clk_freq = sys_clk_freq)
@ -104,22 +257,29 @@ class BaseSoC(SoCCore):
def main():
parser = argparse.ArgumentParser(description="LiteX SoC on Alveo U280")
parser.add_argument("--build", action="store_true", help="Build bitstream")
parser.add_argument("--load", action="store_true", help="Load bitstream")
parser.add_argument("--sys-clk-freq", default=150e6, help="System clock frequency (default: 150MHz)")
parser.add_argument("--ddram-channel",default="0", help="DDRAM channel (default: 0)")
parser.add_argument("--with-pcie", action="store_true", help="Enable PCIe support")
parser.add_argument("--driver", action="store_true", help="Generate PCIe driver")
parser.add_argument("--with-led", action="store_true", help="Enable LED Chaser")
parser.add_argument("--build", action="store_true", help="Build bitstream")
parser.add_argument("--load", action="store_true", help="Load bitstream")
parser.add_argument("--sys-clk-freq", default=150e6, help="System clock frequency (default: 150MHz)") # HBM2 with 250MHz, DDR4 with 150MHz (1:4)
parser.add_argument("--ddram-channel", default="0", help="DDRAM channel (default: 0)") # also selects clk 0 or 1
parser.add_argument("--with-pcie", action="store_true", help="Enable PCIe support")
parser.add_argument("--driver", action="store_true", help="Generate PCIe driver")
parser.add_argument("--with-hbm", action="store_true", help="Use HBM2")
parser.add_argument("--with-analyzer", action="store_true", help="Enable Analyzer.")
parser.add_argument("--with-led-chaser", action="store_true", help="Enable LED Chaser")
builder_args(parser)
soc_core_args(parser)
args = parser.parse_args()
if args.with_hbm:
args.sys_clk_freq = 250e6
soc = BaseSoC(
sys_clk_freq = int(float(args.sys_clk_freq)),
ddram_channel = int(args.ddram_channel, 0),
with_pcie = args.with_pcie,
with_led = args.with_led,
with_led_chaser = args.with_led_chaser,
with_hbm = args.with_hbm,
with_analyzer = args.with_analyzer,
**soc_core_argdict(args)
)
builder = Builder(soc, **builder_argdict(args))