mirror of
https://github.com/saymrwulf/uhd.git
synced 2026-05-14 20:58:09 +00:00
ci: support for x3xx devtest on rhombus
Rhombus is a test suite that is intended run devtests for multiple devices in parallel. This commit adds support for x300 and x310 support. This uses redis to mutex access to the hardware between Azure Pipeline agents. This also updates the fpga using Vivado over ssh to the host machine. Signed-off-by: Steven Koo <steven.koo@ni.com>
This commit is contained in:
Steven Koo
Aaron Rossetto
parent
1efdc3a7f6
commit
64ea8d91f8
6 changed files with 424 additions and 15 deletions
|
|
@ -65,11 +65,15 @@ jobs:
|
|||
vsArch: $(vsArch)
|
||||
vsYear: $(vsYear)
|
||||
|
||||
- task: CopyFiles@2
|
||||
- task: ArchiveFiles@2
|
||||
inputs:
|
||||
sourceFolder: $(Build.BinariesDirectory)
|
||||
targetFolder: $(Build.ArtifactStagingDirectory)
|
||||
displayName: Copy build files to artifact folder
|
||||
rootFolderOrFile: $(Build.BinariesDirectory)
|
||||
includeRootFolder: false
|
||||
archiveType: tar
|
||||
tarCompression: gz
|
||||
archiveFile: $(Build.ArtifactStagingDirectory)/$(dockerOSName)-${{ parameters.toolset }}.tar.gz
|
||||
replaceExistingArchive: true
|
||||
displayName: Compress build files
|
||||
|
||||
- task: CopyFiles@2
|
||||
inputs:
|
||||
|
|
|
|||
128
.ci/templates/job-uhd-devtest-rhombus.yml
Normal file
128
.ci/templates/job-uhd-devtest-rhombus.yml
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
parameters:
|
||||
- name: testOS
|
||||
type: string
|
||||
values:
|
||||
- ubuntu2004
|
||||
- name: uhdSrcDir
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
- template: job-uhd-devtest.yml
|
||||
parameters:
|
||||
suiteName: 'rhombus'
|
||||
testOS: '${{ parameters.testOS }}'
|
||||
knownHost: 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE+SZhHi7YOvHW6xmVGhhZGLtqlZoPkOqGdr5WqnmLBN root@ubuntu'
|
||||
toolset: 'make'
|
||||
uhdSrcDir: '${{ parameters.uhdSrcDir }}'
|
||||
redisHost: 'sdr-rhombus'
|
||||
dutMatrix:
|
||||
rhombus-x300-UBX-0 XG:
|
||||
devType: 'x300'
|
||||
devModel: 'x300'
|
||||
devName: 'rhombus-x300-UBX-0'
|
||||
devSerial: '30A6019'
|
||||
devBus: 'ip'
|
||||
devAddr: '192.168.40.2'
|
||||
devFpga: 'XG'
|
||||
devtestPattern: 'x3x0'
|
||||
jtagSerial: '2516350A6019'
|
||||
jtagServer: 'nitest@sdr-rhombus'
|
||||
rhombus-x310-UBX-0 XG:
|
||||
devType: 'x300'
|
||||
devModel: 'x310'
|
||||
devName: 'rhombus-x310-UBX-0'
|
||||
devSerial: '3138EF5'
|
||||
devBus: 'ip'
|
||||
devAddr: '192.168.40.3'
|
||||
devFpga: 'XG'
|
||||
devtestPattern: 'x3x0'
|
||||
jtagSerial: '251635138E98'
|
||||
jtagServer: 'nitest@sdr-rhombus'
|
||||
rhombus-x310-CBX-0 XG:
|
||||
devType: 'x300'
|
||||
devModel: 'x310'
|
||||
devName: 'rhombus-x310-CBX-0'
|
||||
devSerial: '30796C2'
|
||||
devBus: 'ip'
|
||||
devAddr: '192.168.40.4'
|
||||
devFpga: 'XG'
|
||||
devtestPattern: 'x3x0'
|
||||
jtagSerial: '2516350796C2'
|
||||
jtagServer: 'nitest@sdr-rhombus'
|
||||
rhombus-x310-WBX-0 XG:
|
||||
devType: 'x300'
|
||||
devModel: 'x310'
|
||||
devName: 'rhombus-x310-WBX-0'
|
||||
devSerial: '30C5BFF'
|
||||
devBus: 'ip'
|
||||
devAddr: '192.168.40.5'
|
||||
devFpga: 'XG'
|
||||
devtestPattern: 'x3x0'
|
||||
jtagSerial: '2516350C5BFF'
|
||||
jtagServer: 'nitest@sdr-rhombus'
|
||||
rhombus-x310-SBX-0 XG:
|
||||
devType: 'x300'
|
||||
devModel: 'x310'
|
||||
devName: rhombus-x310-SBX-0
|
||||
devSerial: 'F43D13'
|
||||
devBus: 'ip'
|
||||
devAddr: '192.168.40.6'
|
||||
devFpga: 'XG'
|
||||
devtestPattern: 'x3x0'
|
||||
jtagSerial: '251635F43D13'
|
||||
jtagServer: 'nitest@sdr-rhombus'
|
||||
rhombus-x300-UBX-0 HG:
|
||||
devType: 'x300'
|
||||
devModel: 'x300'
|
||||
devName: 'rhombus-x300-UBX-0'
|
||||
devSerial: '30A6019'
|
||||
devBus: 'ip'
|
||||
devAddr: '192.168.40.2'
|
||||
devFpga: 'HG'
|
||||
devtestPattern: 'x3x0'
|
||||
jtagSerial: '2516350A6019'
|
||||
jtagServer: 'nitest@sdr-rhombus'
|
||||
rhombus-x310-UBX-0 HG:
|
||||
devType: 'x300'
|
||||
devModel: 'x310'
|
||||
devName: 'rhombus-x310-UBX-0'
|
||||
devSerial: '3138EF5'
|
||||
devBus: 'ip'
|
||||
devAddr: '192.168.40.3'
|
||||
devFpga: 'HG'
|
||||
devtestPattern: 'x3x0'
|
||||
jtagSerial: '251635138E98'
|
||||
jtagServer: 'nitest@sdr-rhombus'
|
||||
rhombus-x310-CBX-0 HG:
|
||||
devType: 'x300'
|
||||
devModel: 'x310'
|
||||
devName: 'rhombus-x310-CBX-0'
|
||||
devSerial: '30796C2'
|
||||
devBus: 'ip'
|
||||
devAddr: '192.168.40.4'
|
||||
devFpga: 'HG'
|
||||
devtestPattern: 'x3x0'
|
||||
jtagSerial: '2516350796C2'
|
||||
jtagServer: 'nitest@sdr-rhombus'
|
||||
rhombus-x310-WBX-0 HG:
|
||||
devType: 'x300'
|
||||
devModel: 'x310'
|
||||
devName: 'rhombus-x310-WBX-0'
|
||||
devSerial: '30C5BFF'
|
||||
devBus: 'ip'
|
||||
devAddr: '192.168.40.5'
|
||||
devFpga: 'HG'
|
||||
devtestPattern: 'x3x0'
|
||||
jtagSerial: '2516350C5BFF'
|
||||
jtagServer: 'nitest@sdr-rhombus'
|
||||
rhombus-x310-SBX-0 HG:
|
||||
devType: 'x300'
|
||||
devModel: 'x310'
|
||||
devName: rhombus-x310-SBX-0
|
||||
devSerial: 'F43D13'
|
||||
devBus: 'ip'
|
||||
devAddr: '192.168.40.6'
|
||||
devFpga: 'HG'
|
||||
devtestPattern: 'x3x0'
|
||||
jtagSerial: '251635F43D13'
|
||||
jtagServer: 'nitest@sdr-rhombus'
|
||||
91
.ci/templates/job-uhd-devtest.yml
Normal file
91
.ci/templates/job-uhd-devtest.yml
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
parameters:
|
||||
- name: suiteName
|
||||
type: string
|
||||
- name: testOS
|
||||
type: string
|
||||
values:
|
||||
- ubuntu2004
|
||||
- name: knownHost
|
||||
type: string
|
||||
- name: toolset
|
||||
type: string
|
||||
values:
|
||||
- make
|
||||
- name: uhdSrcDir
|
||||
type: string
|
||||
- name: redisHost
|
||||
type: string
|
||||
- name: dutMatrix
|
||||
type: object
|
||||
|
||||
jobs:
|
||||
- job: uhd_devtest_${{ parameters.suiteName }}_${{ parameters.testOS }}
|
||||
displayName: uhd devtest ${{ parameters.suiteName }} ${{ parameters.testOS }}
|
||||
pool:
|
||||
name: de-dre-lab
|
||||
demands:
|
||||
- suiteName -equals ${{ parameters.suiteName }}
|
||||
- testOS -equals ${{ parameters.testOS }}
|
||||
variables:
|
||||
- group: sdr-pipeline-vars
|
||||
strategy:
|
||||
matrix: ${{ parameters.dutMatrix }}
|
||||
workspace:
|
||||
clean: outputs
|
||||
steps:
|
||||
- checkout: self
|
||||
clean: true
|
||||
- task: InstallSSHKey@0
|
||||
displayName: 'Install Ettus SSH key'
|
||||
inputs:
|
||||
knownHostsEntry: '${{ parameters.knownHost }}'
|
||||
sshPublicKey: '$(ettus_ssh_pubkey)'
|
||||
sshKeySecureFile: 'id_rsa.ettus'
|
||||
- download: current
|
||||
artifact: ${{ parameters.testOS }}-${{ parameters.toolset }}
|
||||
displayName: Download pipeline artifact ${{ parameters.testOS }}-${{ parameters.toolset }}
|
||||
- task: ExtractFiles@1
|
||||
inputs:
|
||||
archiveFilePatterns: $(Pipeline.Workspace)/${{ parameters.testOS }}-${{ parameters.toolset }}/${{ parameters.testOS }}-${{ parameters.toolset }}.tar.gz
|
||||
destinationFolder: $(Build.BinariesDirectory)
|
||||
cleanDestinationFolder: true
|
||||
- script: |
|
||||
cd $(Build.BinariesDirectory)/uhddev/build
|
||||
mkdir -p fpga_images
|
||||
rm -rf fpga_images/*
|
||||
python3 utils/uhd_images_downloader.py -t $(devModel) -i fpga_images \
|
||||
-b $(sdr-fileserver)
|
||||
displayName: Download FPGA Images
|
||||
- script: |
|
||||
mkdir -p $(Common.TestResultsDirectory)/devtest
|
||||
cd $(Common.TestResultsDirectory)/devtest
|
||||
export PATH=$(Build.BinariesDirectory)/uhddev/build/utils:$(Build.BinariesDirectory)/uhddev/build/examples:$PATH
|
||||
export LD_LIBRARY_PATH=$(Build.BinariesDirectory)/uhddev/build/lib:$LD_LIBRARY_PATH
|
||||
python3 ${{ parameters.uhdSrcDir }}/.ci/utils/mutex_hardware.py \
|
||||
--jtag_x3xx $(jtagServer),$(jtagSerial),$(Build.BinariesDirectory)/uhddev/build/fpga_images/usrp_$(devModel)_fpga_$(devFpga).bit \
|
||||
${{ parameters.redisHost }} $(devName) \
|
||||
"$(Build.BinariesDirectory)/uhddev/build/utils/uhd_usrp_probe --args addr=$(devAddr)" \
|
||||
"python3 ${{ parameters.uhdSrcDir }}/host/tests/devtest/run_testsuite.py \
|
||||
--src-dir ${{ parameters.uhdSrcDir }}/host/tests/devtest \
|
||||
--devtest-pattern $(devtestPattern) --args addr=$(devAddr),type=$(devType) \
|
||||
--build-type Release --build-dir $(Build.BinariesDirectory)/uhddev/build \
|
||||
--python-interp python3 --xml"
|
||||
continueOnError: true
|
||||
condition: and(succeeded(), eq(variables.devType, 'x300'), eq(variables.devBus, 'ip'))
|
||||
displayName: Run devtest on $(devName) $(devFpga)
|
||||
- script: |
|
||||
cd $(Common.TestResultsDirectory)/devtest
|
||||
python3 ${{ parameters.uhdSrcDir }}/.ci/utils/format_devtest_junitxml.py \
|
||||
$(Common.TestResultsDirectory)/devtest \
|
||||
$(Common.TestResultsDirectory)/devtest/devtestresults.xml
|
||||
continueOnError: true
|
||||
displayName: Format devtest xml
|
||||
- task: PublishTestResults@2
|
||||
inputs:
|
||||
testResultsFormat: 'JUnit'
|
||||
testResultsFiles: '$(Common.TestResultsDirectory)/devtest/devtestresults.xml'
|
||||
testRunTitle: $(devName) $(devFpga) devtest
|
||||
buildConfiguration: 'Release'
|
||||
mergeTestResults: true
|
||||
failTaskOnFailedTests: true
|
||||
displayName: Upload devtest results
|
||||
|
|
@ -25,16 +25,27 @@ resources:
|
|||
- pipeline: uhd_build_docker_container
|
||||
source: 'uhddev Build Docker Containers'
|
||||
branch: master
|
||||
stages:
|
||||
- stage: build_uhd_stage
|
||||
displayName: Build UHD
|
||||
jobs:
|
||||
- template: templates/job-get-latest-uhd-docker.yml
|
||||
|
||||
jobs:
|
||||
- template: templates/job-get-latest-uhd-docker.yml
|
||||
- template: templates/job-uhd-build-src.yml
|
||||
parameters:
|
||||
toolset: make
|
||||
- template: templates/job-uhd-build-src.yml
|
||||
parameters:
|
||||
toolset: ninja
|
||||
- template: templates/job-uhd-build-src.yml
|
||||
parameters:
|
||||
toolset: msbuild
|
||||
|
||||
- template: templates/job-uhd-build-src.yml
|
||||
parameters:
|
||||
toolset: make
|
||||
- template: templates/job-uhd-build-src.yml
|
||||
parameters:
|
||||
toolset: ninja
|
||||
- template: templates/job-uhd-build-src.yml
|
||||
parameters:
|
||||
toolset: msbuild
|
||||
- stage: test_uhd_stage
|
||||
displayName: Test UHD
|
||||
dependsOn: build_uhd_stage
|
||||
jobs:
|
||||
- template: templates/job-uhd-devtest-rhombus.yml
|
||||
parameters:
|
||||
testOS: ubuntu2004
|
||||
uhdSrcDir: $(Build.SourcesDirectory)
|
||||
|
|
|
|||
97
.ci/utils/jtag/viv_hardware_utils.tcl
Normal file
97
.ci/utils/jtag/viv_hardware_utils.tcl
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
# Function definitions
|
||||
proc ::connect_server { {hostname localhost} {port 3121} } {
|
||||
if { [string compare [current_hw_server -quiet] ""] != 0 } {
|
||||
disconnect_server
|
||||
}
|
||||
connect_hw_server -url $hostname:$port
|
||||
}
|
||||
|
||||
proc ::disconnect_server { } {
|
||||
disconnect_hw_server [current_hw_server]
|
||||
}
|
||||
|
||||
proc ::jtag_list {} {
|
||||
# Iterate through all hardware targets
|
||||
set hw_targets [get_hw_targets -of_objects [current_hw_server -quiet] -quiet]
|
||||
set idx_t 0
|
||||
foreach hw_target $hw_targets {
|
||||
puts "== Target${idx_t}: $hw_target =="
|
||||
open_hw_target $hw_target -quiet
|
||||
# Iterate through all hardware devices
|
||||
set hw_devices [get_hw_devices]
|
||||
set idx_d 0
|
||||
foreach hw_device $hw_devices {
|
||||
puts "--- Device${idx_d}: $hw_device (Address = ${idx_t}:${idx_d})"
|
||||
set idx_d [expr $idx_d + 1]
|
||||
}
|
||||
close_hw_target -quiet
|
||||
set idx_t [expr $idx_t + 1]
|
||||
}
|
||||
}
|
||||
|
||||
proc ::jtag_program { filepath {serial "."} {address "0:0"} } {
|
||||
set idx_t [lindex [split $address :] 0]
|
||||
set idx_d [lindex [split $address :] 1]
|
||||
|
||||
set hw_targets [get_hw_targets -of_objects [current_hw_server]]
|
||||
set hw_targets_regexp {}
|
||||
|
||||
foreach target $hw_targets {
|
||||
if { [regexp $serial $target] } {
|
||||
set hw_targets_regexp [concat $hw_targets_regexp $target]
|
||||
}
|
||||
}
|
||||
|
||||
set hw_target [lindex $hw_targets_regexp $idx_t]
|
||||
|
||||
if { [string compare $hw_target ""] == 0 } {
|
||||
error "ERROR: Could not open hw_target $idx_t. Either the address $address is incorrect or the device is not connected."
|
||||
} else {
|
||||
open_hw_target $hw_target -quiet
|
||||
}
|
||||
|
||||
set hw_device [lindex [get_hw_devices] $idx_d]
|
||||
if { [string compare $hw_device ""] == 0 } {
|
||||
close_hw_target -quiet
|
||||
error "ERROR: Could not open hw_device $idx_d. Either the address $address is incorrect or the device is not connected."
|
||||
} else {
|
||||
puts "- Target: $hw_target"
|
||||
puts "- Device: $hw_device"
|
||||
puts "- Filename: $filepath"
|
||||
puts "Programming..."
|
||||
current_hw_device $hw_device
|
||||
set_property PROBES.FILE {} [current_hw_device]
|
||||
set_property PROGRAM.FILE $filepath [current_hw_device]
|
||||
program_hw_devices [current_hw_device]
|
||||
close_hw_target -quiet
|
||||
puts "Programming DONE"
|
||||
}
|
||||
}
|
||||
|
||||
# Initialization sequence
|
||||
open_hw_manager
|
||||
connect_server
|
||||
|
||||
if [expr $argc > 0] {
|
||||
#Execute a command and exit
|
||||
set cmd [lindex $argv 0]
|
||||
if { [string compare $cmd "list"] == 0 } {
|
||||
jtag_list
|
||||
} elseif { [string compare $cmd "program"] == 0 } {
|
||||
set filepath [lindex $argv 1]
|
||||
if [expr $argc == 3] {
|
||||
set serial [lindex $argv 2]
|
||||
jtag_program $filepath $serial
|
||||
} elseif [expr $argc > 3] {
|
||||
set serial [lindex $argv 2]
|
||||
set devaddr [lindex $argv 3]
|
||||
jtag_program $filepath $serial $devaddr
|
||||
} else {
|
||||
jtag_program $filepath
|
||||
}
|
||||
} else {
|
||||
error "Invalid command: $cmd"
|
||||
}
|
||||
disconnect_server
|
||||
exit
|
||||
}
|
||||
78
.ci/utils/mutex_hardware.py
Normal file
78
.ci/utils/mutex_hardware.py
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
# mutex_hardware uses redis get a lock on hardware
|
||||
# to prevent other Azure Pipeline agents from use.
|
||||
# It also provides helper functions to get devices
|
||||
# into a state where it can be used for testing.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from fabric import Connection
|
||||
from pottery import Redlock
|
||||
from redis import Redis
|
||||
|
||||
|
||||
def jtag_x3xx(jtag_args, redis_server):
|
||||
remote_working_dir = "pipeline_fpga"
|
||||
vivado_program_jtag = "/opt/Xilinx/Vivado_Lab/2020.1/bin/vivado_lab -mode batch -source {}/viv_hardware_utils.tcl -nolog -nojournal -tclargs program".format(
|
||||
remote_working_dir)
|
||||
jtag_server, jtag_serial, fpga_path = jtag_args.split(",")
|
||||
print("Waiting on jtag mutex for {}".format(jtag_server), flush=True)
|
||||
with Redlock(key="hw_jtag_{}".format(jtag_server),
|
||||
masters=redis_server, auto_release_time=1000 * 60 * 5):
|
||||
print("Got jtag mutex for {}".format(jtag_server), flush=True)
|
||||
with Connection(host=jtag_server) as jtag_host:
|
||||
jtag_host.run("mkdir -p " + remote_working_dir)
|
||||
jtag_host.run("rm -rf {}/*".format(remote_working_dir))
|
||||
jtag_host.put(
|
||||
os.path.join(pathlib.Path(
|
||||
__file__).parent.absolute(), "jtag/viv_hardware_utils.tcl"),
|
||||
remote=remote_working_dir)
|
||||
jtag_host.put(fpga_path, remote=remote_working_dir)
|
||||
jtag_host.run(vivado_program_jtag + " " +
|
||||
os.path.join(remote_working_dir, os.path.basename(fpga_path)) +
|
||||
" " + jtag_serial)
|
||||
print("Waiting 15 seconds for device to come back up", flush=True)
|
||||
time.sleep(15)
|
||||
|
||||
|
||||
def main(args):
|
||||
redis_server = {Redis.from_url(
|
||||
"redis://{}:6379/0".format(args.redis_server))}
|
||||
print("Waiting to acquire mutex for {}".format(args.dut_name), flush=True)
|
||||
with Redlock(key=args.dut_name, masters=redis_server, auto_release_time=1000 * 60 * args.dut_timeout):
|
||||
print("Got mutex for {}".format(args.dut_name), flush=True)
|
||||
if(args.jtag_x3xx != None):
|
||||
jtag_x3xx(args.jtag_x3xx, redis_server)
|
||||
for command in args.test_commands:
|
||||
result = subprocess.run(shlex.split(command))
|
||||
if(result.returncode != 0):
|
||||
sys.exit(result.returncode)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
# jtag_x3xx will flash the fpga for a given jtag_serial using
|
||||
# Vivado on jtag_server. It uses SSH to control jtag_server.
|
||||
# Provide fpga_path as a local path and it will be copied
|
||||
# to jtag_server.
|
||||
parser.add_argument("--jtag_x3xx", type=str,
|
||||
help="user@jtag_server,jtag_serial,fpga_path")
|
||||
parser.add_argument("--dut_timeout", type=int, default=30,
|
||||
help="Dut mutex timeout in minutes")
|
||||
parser.add_argument("redis_server", type=str,
|
||||
help="Redis server for mutex")
|
||||
parser.add_argument("dut_name", type=str,
|
||||
help="Unique identifier for device under test")
|
||||
# test_commands allows for any number of shell commands
|
||||
# to execute. Call into mutex_hardware with an unlimited
|
||||
# number of commands in string format as the last positional arguments.
|
||||
parser.add_argument("test_commands", type=str,
|
||||
nargs="+", help="Commands to run")
|
||||
args = parser.parse_args()
|
||||
main(args)
|
||||
Loading…
Reference in a new issue