saymrwulf/uhd
1
0
Fork 0
mirror of https://github.com/saymrwulf/uhd.git synced 2026-05-16 21:10:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

tests: enable automated streaming tests on X410

Browse source 
Enable automated streaming tests to run on X410.
Disable tests on non-X410 devices for now.

Run only DPDK tests since the setup can not withstand
4Rx 4Tx and 4FDx in non-DPDK mode in manual testing.
And we need to enable running these tests for 100GbE testing.

Adjust streaming test thresholds to new values which seemed to work fine
in manual testing. Might make them tighter in future based on more data.
This commit is contained in:
Virendra Kakade 2022-03-28 14:32:05 -05:00 committed by Aaron Rossetto
parent 218a75e4da
commit fe2e24e79e
6 changed files with 39 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
.ci/templates/job-uhd-streaming-tests-beauty.yml
View file

@ -17,9 +17,15 @@ jobs:
toolset: 'make'
uhdSrcDir: '${{ parameters.uhdSrcDir }}'
dutMatrix:
beauty-N320-0 XG:
dutName: 'beauty-N320-0'
dutType: 'N320'
# beauty-N320-0 XG:
# dutName: 'beauty-N320-0'
# dutType: 'N320'
# dutAddr: '192.168.10.2'
# dutSecondAddr: '192.168.20.2'
# dutMgmtAddr: '10.0.57.13'
beauty-X410-0 CG_400:
dutName: 'beauty-X410-0'
dutType: 'X410'
dutAddr: '192.168.10.2'
dutSecondAddr: '192.168.20.2'
dutMgmtAddr: '10.0.57.13'
dutMgmtAddr: '10.0.57.29'

32
.ci/templates/job-uhd-streaming-tests.yml
View file

@ -43,20 +43,20 @@ jobs:
archiveFilePatterns: $(Pipeline.Workspace)/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}/uhddev-${{ parameters.testOS }}-${{ parameters.toolset }}.tar.gz
destinationFolder: $(Build.BinariesDirectory)
cleanDestinationFolder: true
- script: |
cd ${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
sudo ./setup.sh --auto
sleep 5
displayName: setup interfaces for use without DPDK
- script: |
set -x
export PYTHONPATH=${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
cd ${{ parameters.uhdSrcDir }}/host/tests/pytests
python3 -m pytest -s test_streaming.py -m "not dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \
--addr $(dutAddr) --second_addr $(dutSecondAddr) --mgmt_addr $(dutMgmtAddr) \
--uhd_build_dir $(Build.BinariesDirectory)/uhddev/build --junitxml $(Common.TestResultsDirectory)/TEST-$(dutName).xml
continueOnError: true
displayName: Run streaming tests on $(dutName)
# - script: |
# cd ${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
# sudo ./setup.sh --auto
# sleep 5
# displayName: setup interfaces for use without DPDK
# - script: |
# set -x
# export PYTHONPATH=${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
# cd ${{ parameters.uhdSrcDir }}/host/tests/pytests
# sudo python3 -m pytest -s test_streaming.py -m "not dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \
# --addr $(dutAddr) --second_addr $(dutSecondAddr) --mgmt_addr $(dutMgmtAddr) \
# --uhd_build_dir $(Build.BinariesDirectory)/uhddev/build --junitxml $(Common.TestResultsDirectory)/TEST-$(dutName).xml
# continueOnError: true
# displayName: Run streaming tests on $(dutName)
- script: |
cd ${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
sudo ./setup.sh --auto --dpdk
@ -66,7 +66,9 @@ jobs:
set -x
export PYTHONPATH=${{ parameters.uhdSrcDir }}/host/tests/streaming_performance
cd ${{ parameters.uhdSrcDir }}/host/tests/pytests
sudo --preserve-env=PYTHONPATH python3 -m pytest -s test_streaming.py -m "dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \
# Disable creation of __pycache__ files using -B to avoid errors on clean up during next run
# as using sudo below creates them with root priveleges.
sudo --preserve-env=PYTHONPATH python3 -B -m pytest -s test_streaming.py -m "dpdk" --dut_type $(dutType) --test_length ${{ parameters.testLength }} \
--addr $(dutAddr) --second_addr $(dutSecondAddr) --mgmt_addr $(dutMgmtAddr) \
--uhd_build_dir $(Build.BinariesDirectory)/uhddev/build --junitxml $(Common.TestResultsDirectory)/TEST-$(dutName)-dpdk.xml
continueOnError: true

2
.ci/templates/stages-uhd-pipeline.yml
View file

@ -27,7 +27,7 @@ parameters:
type: string
- name: run_streaming_tests
type: boolean
default: false
default: true
- name: build_sdk
type: boolean
default: false

2
.ci/uhd-pipeline-pr.yml
View file

@ -40,7 +40,7 @@ parameters:
- name: run_streaming_tests
type: boolean
displayName: Run Streaming Tests
default: False
default: true
- name: build_sdk
type: boolean
displayName: Build embedded image SDKs

2
.ci/uhd-pipeline.yml
View file

@ -32,7 +32,7 @@ parameters:
- name: run_streaming_tests
type: boolean
displayName: Run Streaming Tests
default: False
default: true
- name: build_sdk
type: boolean
displayName: Build embedded image SDKs

17
host/tests/pytests/test_streaming.py
View file

@ -263,16 +263,17 @@ def test_streaming(pytestconfig, dut_type, use_dpdk, dual_SFP, rate, rx_rate, rx
print(batch_run_benchmark_rate.get_summary_string(stats, iterations, benchmark_rate_params))
# compare results against thresholds
dropped_samps_threshold = 0
overruns_threshold = 2
rx_timeouts_threshold = 0
rx_seq_err_threshold = 0
# TODO: Have non adhoc better thresholds.
dropped_samps_threshold = 50
overruns_threshold = 50
rx_timeouts_threshold = 50
rx_seq_err_threshold = 50
underruns_threshold = 2
tx_timeouts_threshold = 0
tx_seq_err_threshold = 0
underruns_threshold = 50
tx_timeouts_threshold = 50
tx_seq_err_threshold = 50
late_cmds_threshold = 0
late_cmds_threshold = 50
# TODO: define custom failed assertion explanations to avoid extra output
# https://docs.pytest.org/en/6.2.x/assert.html#defining-your-own-explanation-for-failed-assertions