From 78a29aebbcbd0c3b6dab734f221e0f3bf1e24c97 Mon Sep 17 00:00:00 2001 From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com> Date: Sat, 19 Sep 2020 01:43:22 -0700 Subject: [PATCH] [ORT Mobile] ORT Minimal E2E CI (#5200) * Modify the ort minimal CI to ort minimal e2e ci --- .../test_voice_commands/model.onnx | Bin 0 -> 11605 bytes .../test_data_set_0/input_0.pb | 1 + .../test_data_set_0/output_0.pb | 1 + tools/ci_build/exclude_unused_ops.py | 2 +- .../linux-cpu-minimal-build-ci-pipeline.yml | 59 +++++++++++++----- .../build_full_ort_and_create_ort_files.sh | 39 ++++++++++++ .../build_minimal_ort_and_run_tests.sh | 34 ++++++++++ tools/python/convert_onnx_models_to_ort.py | 2 +- 8 files changed, 122 insertions(+), 16 deletions(-) create mode 100755 onnxruntime/test/testdata/ort_minimal_e2e_test_data/test_voice_commands/model.onnx create mode 100644 onnxruntime/test/testdata/ort_minimal_e2e_test_data/test_voice_commands/test_data_set_0/input_0.pb create mode 100644 onnxruntime/test/testdata/ort_minimal_e2e_test_data/test_voice_commands/test_data_set_0/output_0.pb create mode 100644 tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh create mode 100644 tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh diff --git a/onnxruntime/test/testdata/ort_minimal_e2e_test_data/test_voice_commands/model.onnx b/onnxruntime/test/testdata/ort_minimal_e2e_test_data/test_voice_commands/model.onnx new file mode 100755 index 0000000000000000000000000000000000000000..bfad5fafadeeeea51b66260e99d2d698100528c9 GIT binary patch literal 11605 zcmb_?2{=_>_b@VEL#9k6VFm8j#)OngR5E4AkXf@rD9IclLnRGF1G#7Kt*8{G zK^j#`X{1RdDWBi`z2E=;{h#;!p6@=-xo7RY&N^$Yz4jXRy-Qd~nzJ?_$|u?|B0M}n zNx(qYR99bJ^tVA+XiP*zXtWlOA&*7=a*h^9C^&pwY>c;ow4k3)bWDK1692sMyZZ0n z+9DjjmIa$Y`8Fh?#R%s0T_KR7&4?{6sDLZ9;|q4bQuXaB)_hekxKja{eb=pPUs z6CAVN+rUC!N#ySb)dlT+VuAvqv_(1mJ_*6m3Ov8D^5y)yuacJhL;9E~pYZ5)5z*t~ zi@5&DAc9wLnC~BLapM)%@Y`~aQr=-GrXMtn6b3szboqBOQWx3@E_Csk4t!$m-GK1 z1!vB`D@9a5bdb-w0PpA+pQxCB-!^}e{Rg)BlR{l+(H|T+WB-9g{$j_G7~kU20ea(p z7akBg?iONyjD3TB#+^j^&(WfQ(AaSg;H-%q->a&D0byb4B6h#Q_T!+oI7jezb6NlH z2ig)Ge&2u?pGiDCiF3<&dH+UR{ehmy5%39J7xWK)5{KjC7ZV)k6B8T}uFn4(uX5fA zh8lk``;Eiz9OHvyf>i%`Q1y42#Qh?|bi;!Eq9URr*2Hk?dH<6C4~aRFziZ=d@Q;3f z=sQ01r-stM-2?o+W5$;uBFa15CoDio>^BX)VJ zeZu`!t^9w#IJTVsMg(c0KLuA3T0E{;m`{TGUx2pkpFwBrug`d7;FIP=2SmpH7Oi|j zwTs1tC54qFdDRoe`&+i7W_Ezov^CQ8f!e5n{~lS+e~Oac<%GJHfV%GkM8dg^{Z3aPbe)3RmLki+q67-}^E@jfgd?rF_< z_=r8y5|cx|Tjrzs2Z?B3*$i~GVkZs8*|c2Wfj;2>03C0wr;ireqq`z`9YJ-5AC&;VKx8lSc6duJpq;0VHNWLi*3u z(m4wp>4o!i$ey4iblChetWOSxx5a;uvz1xYb=!1W{=5)1oeZWG0>_DOLoDsDd4S8d z730c~qp-QAnGQeIgZz65S-tf%%8Ff%PtVOm3Tfs<#;*vi=uklBKkt)IET9rTv!F?C zBr(z=bnDy#k|EGQRX2WySuu+A!sjds3+5qS3t=jK<{^}FoMC!yFQq5|p*Z z9GO2@MpGSy5Fc!!jhXc*x8VbcKUsh_R65gNS#}6rKTGu#$_Q6=9nA1Ji#`d}AT^T= zdOY|n4Edpj;vQ4$yJ#_Z5~YT>EpsDo_9t*olnQl;Ka12)o=0tCS1C$P zfCnm5(2c2es9k3>mVH!*f<)G%K;IR#C9wrHKNLq%XKm=Lrqy&(e?AI5ltrd2IYaNd z2+~*md(eT|W$=sddwh3uEK16$MC#hwu(YFu_Qy3LtHn>?&l5s4w(t(DSQbvNx2Yg$ z#}K4=*o_?Pc1IIGTBF^{1@N=OEizQv3e{qZ(AP`H>FUBV_|#|`y}jrRihD1QhF7ML zl}%MB&}%Q!>0U~$4Wy8Raz8wuT0qk0gix}D4;gAyQsW7>sCThAHOy_HZE8CxyirP* zdF?{y6W$W>qiXb}(nZK8u!d$YdH}8F9zY{GJ7|AH1_cTisbkj#&=oilX8-GIJu)WGd;PXbmbq&r5kSN8z_E!F17=7?jgt$2$2x0$w2o^x2B7DBUNQ9FaK$ ze`GC0RomxNujxB!maacy7dSvXFOk0ZX$RY;jPJR8Bh=`xLK--g^vx@CvPdwS*edFw zeQzsikZAV=`^m=g_iqR>-TWoZF2Tij#rViDw*zMLyG(u1S4a~ku%?iUoU({YIvF|Fh z@SIQc8tb5*Z#`m^EzoDJ7>Yh&G-u#2JE5kKZzdPX z%B&}@QC;|Z%3;(M)lE_l=~Bg;M^L!a9LBt96XYMxLBiiTBuZBVZR^ND4~I5U^jrcd zC)Og77H@QX_G-GI=_nn#u#G;R@P;&Oh=4as_fx^|r_ild3_o^^$HI5kG(&SKsoZ2s zo$F)h;)Tj6!`2s#d`m>ZHA(caUoBZ(6+~RF1>jUmKa`U@7o{r45Fra?`b{$pIk~DJ z&lVTraOeV8O`ajkuV)dHJ(KCLrFqD5=6rN%*J&DmUK)KWDx?VqSQ=0uLbn<^(gS(Z zPy}BY5x=rc^)o-fN;p6XMUv>zx9Ee2rR;e^J&Isxp%BG^?lhC0@HdM8(58f{3 zN3lB-kh!EN6+f4T&TWsR#{%Y{jozD35N{Y=VtgC4yz3+Z?PX}$acTN(!zmi=Q;jn3 ziPQcpR~oE-6pd~303(*4GoL{8%{SFykZ@NVKAtpE;n?pJ!0;%yfEFGnnq2 zAwsSHO9s~7@_T9uvMa1(gD<9Fo7`|5vqBYjz7GNu`uM>^zfOh|;KBAPOk|x~3z)Rm z;h5x11>Vx~__|RFI}p8>_08&Klm|q#J(G$GKu(yo?`D;2=4Jdq%J$OGSq z-pxYiU~K(=WPI%+;qmM!`CE1rPVx1mRZGqTADdLNwXT!iG5JNB@^xucM;$x{AHvAF zn9`~!n()z+o|#*YCU$jDX}S-QkyS{|Pz|MLU!v1;6w&hjWRyL#3#s(k!b8K0$!K;C z{qkW9>aL0>Pwq`;`K))+y}H?`ul6%uGm=YUzp%(4_8>YTz7;)Kp@7OZ%c0F3`w-{Z z2-xJ^NKZw|LYGO&=$EDd^^m$vzdaoRrPm%aE!NxUWxkViqUI@TFTI>}DBMF5GL=+Q zdM5&_FVfZCxm5SHA(Fe{ML+7fqMQpYbi}Bco}79J4WHdeJri#td)?z`W<@2+o7jlB zdXY$-)j)HPX`!WcW=Lxop*?%=AltxpN_SO(x7o*lN>4sbIa`7%E0v*NST1rAIY=|Q z%SeiHHNB;KiTg8cIwc2A(mih#=ss;i<+eZ=W?e&vE27~{-V`!0LzL>9h|+ak`M5=| z3WZ6!k%f;d=qt7cmb+J>tqbqaZIW_SaBUKD$eKVeSesI4v=HerCsAd<4ceShL-}81 z&}SDjsK<_ZC~Wt2^w#n+t>0V+8eNi*oB2|TPoz+_H{ImO^rK{xlP6svT!8X!_0d|t zL=;#Yf#$q9L(SAa6HWUy^w+gibl0(krVED>C6f}$*{MMj68ccZJpL5`m1DBs2_)L^R*<=HhK-I?b>1(v7K+N50&%J$G%m#gV>uMx znN*ZExB)#9S0|CY`pCr1ncC(?BL2GFXzgoBuue3Oh+o)^oPK6g!%v4`z}5n^f1?~t zVEE|CqGoVEb1fN?ttUwb?nA!Jhj5d%AF|y2kb0$$!Hoj}@F%|@dRZt%zxypgJ4CJ{ zBO^O_PpzH?^48E@kGl!JZjS1vl_IN&`RM#`G~!hYMOTIw(VN!0=*7F6;4!o7WCf9; z!b>W!Teb_iKiC8J4OyVe#~&fX_p#KkaSx55ooLC<9>gCv0TsU}rR#Dw(>v3eP-}A| zIdpD6J-+rHaTfDJG3V|=t9Wl()hI`mt0iF3-M#c%b`|yV%BFS><;ZaK0o}OpHZt=X zpKo-9QJ4#ZB`fD$`D0RJs{>uuqeQY08oSH{xZ)u@_#)iL72ioqx2L?+` zi_aG2i(T+l-x|>MD357>a*MH_R1cJMPB8bGd(3DOVb(J0AW|+Im=%3v&J8|eyfi1_ zXGvkK7rMh*|H~;#+d-I5QpiG-=dXf&jMw`A?Kr9Z)I>P!cM;E-Z1o$$)BOi=#d3f z>!q$oR2G)wp)WJwZq;0npIioTkuoU!Ax5hAe#LyYZNSufGtQHV!k1I;;?p^*@Yu^I z!04$XZn!8-atnL#1>-HCB5f9wWp?6rML{CwEQ%Ghg|PL$Z=fq*k?_d7;PT*k?4vSO zvSULlFkPw$J!QUt_5uJF4t3*_is?lEqz_o*9SrW(zTi&oZw0xxeMt{*I9v^jV0V8M zoc$=HDQn;wumE;sSbQdd#`AFebBG<)&f$1{Szs)yNZhPD@L-Avd^aZxSFZm8_M4iL z)yL$>WDP^Oqu+uU?7NTmv|k2rlMT>K`32rrmg32+24oD+BoTab*(t@z_{ifC3sv1L z-1W;6e^xexehsSupL!E>xjF~*Rh$6tjwLhA`bun_rzx3pF$=gZZ~#Z5ClKpNO`!2= zD_i&?3@BMRf(I`;v5r?BNZT?3Zt!Z5_JLYYs#d`Tj&H-6uWasKxxn6~{QE)1G){zzMaXuC@i zhsN2k(oF$I-Ae;~N#gKPO+8-Ww1s#)vjZ)O*5tudc`}D~VDFdufb~5KKD|sKgB}k+ zhCPSe?0p4mYJvYK7q`zpj7>ZbfKGvMqN0_EorY53 z=lCh)OxPj3u44;))%ycKJIb2#W%|Puw{_%c%q}44o=zm*DwB9!ZL(HF8cKaL!}ERT z!%MSI1F_ND%p&)b;QS>>W(PSz&BkJIdF@s_#A#%W`Oaapp&G_rRhameZDyisGl;rH z0ywt&3cl%d7nh5glHzNF%)B~lc)U9iuCT~uhCgew)+!&_?o1!<$*Y%`X=lx#cXc|R zDU(31TNU7``G9zT*R(h`cn|cab>aCF(y&MDMl62GnYDZ>0x0QZjFawJaQl`(^(|B2 z>7bX4^-W)b+9ctL{>ONQo-G`6y@+R6Pi2iN9oeG_XK|InY#3O421|g}tPE*|4-Q zvVBJZ?ok^7S5Id^uN|_GhuwNMHR1`|URaE^+Xcw4R5$WvUMhUMY9evDeiSG*+yGJ| z6>N-_J&|kg#P+j_8THrc&|zp5xmROKc#cUzt#>zZ!J$!LKbXPYvM37XUb2VUiU@A* zT8=~OJW28*6_8--2R?aJg8rYM83*xd_H?`sN#5{=N!q#=uU1Mn^V??!tEW|h_5F>& z)7lD}HP?bo#u&(Fjh8|5HZczqy`jsqx8P^WQ#{SgoR!ZIFt2}p0dpU(0~d^yp`DF1 z+|g(P%N7Wc`;X0G$qPrE@B)(apWE2>I8D5G@G)C^w~1*PW?-kO4|68-5$Hc{1LcD) zNYiZ_7BqfkHcPiKDiVUk!DtoWi1LBxJzXS1;hCov08d&{JjT|wY0hbxakq1+{0bgST`#Gk7 zRMp2o?@PNuNdFwT*~69`9XkikP2P-sMLU7Z=TRK}#2C1KNP(2Gg~~IX*#$!%fo9xt zW*BP%?Gj5;Q?(eQoK*7lQ!V)Ft_R~E3PLHfesJyR7kp(+9*)e@fZ9?zp!e`eoO0wa zRvXnP-9vz6d{ZDR`6Izw$r5(sSw6BnQJWO+)WU}yonY|c7?Nap8Z0=I!)-Y}m0VIc zA@Q&CEwraz#OIdCl2vgVfX~*OcwomH;O_DQd-Y0UuyisUiI1>&6=eW-f7k~sJ2w&( zk&bXkMN$$x)|7j(2t1j&hgsF8irwDHlJ;_Yaw*dmAGP8K8Du*a zITwVT*EzyPu4_p{jxq^)U5I5K&ts~(=7Q-jOR<~wWcd8292~VNWn<%F;jJa-+193H z#%;kv;On!QoY|BH0&;x7S{p6)!Gh)K;6b_#ng4@=J?>Lr?!2imBruRX+6jqNf*dG`aKdduTZnLI4=xVQ z1s{i>VEG3*!1;+M^th@?3^%@OQmHk;PJ8^hD-RZ6+gx5Ev#P|xbps&$Ch6dlc^j_P z+)aX!2|IVC1WC{E#G;#)!sk^g(5=ded^B7H25okMqp%QM|Nayh{^|xQ`Ms>`3Ty1U zHx2C7XaT`(pMewKAZVG!1MA9oF%w`5%??56S33#2Wu5}h9Nb~U8v|zhp;OGPa$WMH zeIilPif2wH&nNCe;%q_M2XLZa8H%@UAT=r%n`)K!fM`iOxUV<`-ah9Jh8QDk*7uUV z8M~F-d)p5ddkN#UFD8LW*_up}^B@zs@&e1hataB$Pzb75u4W<=t?&zzO#JNSMRux} zBXHU%0f$dcY^spIh8I8FNJJkV0|HfbxXe%tiUsPykdHc~&sd$r)bNtEx!TY%^#cf$ zSqMWbw=;LI-(khhrQ@q-7LkqD9Pxuh9q9iq892N)BQl@6nF@VA(&;}L|9H0pJomnY zRW<#YyZXt*A@?n?cPV0fg&oNf*A=kz0|b_lgJ5^tQE=Z>m6*)5f)KxA8V{6!6vdq& z)J+l|Jhl)jomU6R6*9o%{Zrt1>p7SjN5Pj+Z+vfF25_p{2l8j_1salVIKIgWdI~Ow zB~}&8J9Y$r_nt*=e0c?|O1Hw`hj$rKojN8-^E!9nQaqm1aRxhd-vqNX3B#61fb=y=J_fOja4+@sW54xCAA7K>+A%j3;Xa{J|p~ci8;Acs}8>gHsZoAvKS&qEgSLLsB%!+RR9Ki zuj4DPS3{njL6G~{5v)_~VP@_92pHu}02vFAr&OBQ*Qa8cYaC zB6j1oO|zLWxvrZ9{G5cjt)vxq=;VVtQvxmIHY$)G(hdMRKF1SkE#T;=9_an9MnZmR zgAnlO)j53N z(RH9C^%bXg6tP~r%b6G7%*l!A3golfFi76619)70NKCQ|+_ou{J-%Q)G`6vVy_1jN z`CB&QxTET%%Df1yQ`!X6SUdQ$`yhV(qzp$936d#^n09?RaC~VXyegf5RU`R`j^G{~ ze?gAuXzT@DhKj(AKEcPG=EKvw>|odD>#Umn1zgE<8hjC|0nQr&p;j9|5wD$s55D6k z$7?2%q5YRYn;1hH!&Tr#VJq0&b^}aOZnwBt&IJ>0W`I`*)9~J1awPK6RbX1wfKx+k z$eaAxaMkb{g1x6f+f_9<)j*Ckag2!`=M?}|LZoSt3ASy04dnWD$?^_4rf5Ma)>k$q zd_7w%a*G%o8yC&ph~&8P3v^+Q*$U7;$rUtgyapO?36b(H2dpQs6qgwY!T}c#_TFL{ z-1TxlSP=XZM2sLqF9csuDHyvVL}Us%#A*LVP?gw@+wkJO86`?|z1oqrv0NkhUVl!Q*zz`L2 z{A5%^jbMH=$JW<4V*x7b02bdMX!R-n)M8ylGsT0A|tj%4}w+ZZD8}USh0|dz9u9)uLtOzOop+RJH*O-JVNqIF@j9`z#`f zdq98TR2Z{gm{=axR-KBc z+#O>&SH1^NEBK+hX%brd6z~~Wg6ji<JfHU%R0MuC~WK@|KvXa$dt=T*c0NV0HqGVr1-cq;P-IN@S8 zMw!-uodK(|{a~=FH|+Q@3hch?5N^LL z%vz^K49oI?sYN$lcjF`;*uDTR*=h{YGZB)Uxt|FEhkT;hYo&FArCB<>#twqpf z$N=Y@uE!rX%z(dk_>uhiKR{+v7Gc;V3+YSD0z>Io&BZ}SFg?Z z#JUlt|MX8)s;Tr>62}M<(L8a_DUCLM(N_{Lp!m?r;EUK%K)g?_r#X@_e`x^7ZV~} z#Dr%mf~6@60GtOv!Qg(3dUEl@ZVmk7vjWl6$!Ax!<>GpqH1Mrh5}x1T z1V;v*gMCXZ;n3%0KwTym?8}}5e>F`7ch|DGPNxFzRx8DxC28!AU2j1qSwlABn_TA$ zE+ppkN+v4X0BQ~kVz#gW44qleDn{>TU%VE8S3A~#mr`-;nAR{DFGoqj4h^#F{vZ>V zu0|G4J9u^{)Q0V|jA6|(4%~{KF&0sBP+zp3y{!8h4CmRyU(9TH-tQ;tSUe5h9+M|> z&7;iZIz)U1JQy<`sQnqO?2X*_;)M-%dxK4C|N6v(a5H9){vgbC2JAZ|NuV|f_`=zCBCO1S%y zP8|ox9pHx&>!&e&d+o8$Krz0e+KtVB*b&n^FTpRz+Iuy{H&MI?wiJ#sI*f-aV zq&Y#TaFhk7uWbZeK`$J%APuZ9dB$#7C=Wf4Pa)?%kJqlA`Xp+1AKPxU8h^4mihKG3 z!QF|oiGzv>lxizut`=~Jk@a-k`&g2=H0FW^p(|kGYYTX3l|Ed+%8}*So!r%ak~ngI zC05Zm!3bGn?47uu4S6BOi1pNir-$w_3ER!!hmm^5R>%sD9lVAca&2JW7GWUoGr*kM z@D`-$Oo9z2f_T>Do8TFn1nS2Yz;6#c$s^b4#O}sVypM$>^|CZ+3jfTyG~NeSR9}G` z3J=&vH&luH&tPa@BMBu>PJk*IBS8K>gngezaae#ZXz7;4A=e#=nP4a=>x5vYdLAA< zEy=p+s1xJNGr+e>55nEzr0B&MYqZf1zo?rGTk819lbcBf6uFxQ{u3Z}!WQlPZ(2 zM|lQ#&a_~D-&MpfQII_95g{FF&0wWPIQyt_9)9;$8>aXQVq4n^+!vn1bgWT?bz-)x z0q$V7y{-Wt76`yNuTOY?oHj{caR)rvyvCw@gYemEvDe^Xl@j5)nUiIvTI7XJ5J}wp znmzlL4_By*fDEIrz(>89S#rUVNTTVGXO}d%oVT2`W)uVQNEy=jWhpV4KNr4_3L#Y= z`Jnqscc>D02RtI8qJ|)&^QQ4_;bOH9nHYKObgU& zyaOKHKFn6Hec$8m0&rpO9Qg292$4OK1~#9v#CJA} zSsZ(5Kqef{u^4?~PdpiYTqCdzL}*{aNgpoaE#liO%&R|RHeZU_C)Et1`PHG^&oc0C zX%hH0M-aZ1`-uDRHiM61_cIuEn3cH0WfPmFNkcpd6v6Aj*8ufXeD@OZvPxOVNdvd+dT5oBOXVc&%y%l-+>hlTI6=S8oa2Q&o zc)ZR^Kt`~B#vD?XAC6R~Y|i+vkb-lQCh!7dyxn-Qa;1W0LITs!q0Yy$q~aJ;18P zT?MX@{zN56i(7rC6bNXi;aPb6oFS$N3sx0k{=>4w^MnxDsKE>Rl;5zc8n-f^2JV1& zI@-Xr%!at!PQ-1$EE(`seLTME!X@uEGfjd5NZB6=mfRL66;q^0@W2P4 z?k)$d<0s%cnH=!=(*#g0wC3!DHh=8D<25*vTmVGW$A3?q_li59b_?9vUWcCt?*?O$ zeBc?Umkr%tz=nK|<|Z?q#D3vKd~KHnlscV)XP6$r$JfilgZI?fscU73=%;DqgvBog zUvP)PZK5RngAI^LZUHN*#o*wUvuwZvMUv?;8z?p#;(~3@*m=g^ar8?iI3?QC;zZXA zM(50ae09lJywbD_96jj-Z^nMbVyTMkH$hn@TiO_(tli2E-Uj!-3 z>P}$S-;L{)Rlpl-3few(VXuWrIIeI5_B$p@vJWbfrFGHl+TL8y!yVsK-Htdj^%vmo z6(c-1UUJ2M*^=E6GQ_BLHsq`P0q%xsKpx*>CdS$jcvSM>nACAw4{Qe`Ltnv~@tkNW zvY$I7CI_w(Z7^7I2Jq|^XAi%a1^K>Z0?*k!V8pKz4DpJQlilY5-=_0m!3;+r7bHl; zcXi`)q02FR0$`iMJhHr58D8?K02?HV@jkB#;G9yA%WrtFmM*qraY;W;O>u(#@d z3&nn1erB7+yeL(uTr`;k$~EF;dL~3C`4F}YG~uq!ID?hGri`Cc_knp6#{I+-K|7Ib ze2!}cHtTAV?K?}rh+!G#>um)h<9$*4PVWTwyj<}C!v)0p|I!$|h{sd$pY~!2j+ij7 zv@nNHn0NdV^c_e2*_kK(uP7yJeh!bMn6<=sH)F8wZ#~PpVWEAkNf6cCo3Wy61h}Zq!uu`0V?Q0w-kmtz!ORJ-9xL;7b-#9{rGx=W;VWDdR Qd}3mwf;R+2MN9JjFCIwoqyPW_ literal 0 HcmV?d00001 diff --git a/onnxruntime/test/testdata/ort_minimal_e2e_test_data/test_voice_commands/test_data_set_0/input_0.pb b/onnxruntime/test/testdata/ort_minimal_e2e_test_data/test_voice_commands/test_data_set_0/input_0.pb new file mode 100644 index 0000000000..2992c9c839 --- /dev/null +++ b/onnxruntime/test/testdata/ort_minimal_e2e_test_data/test_voice_commands/test_data_set_0/input_0.pb @@ -0,0 +1 @@ +Binput_1J@*.??@8,}@ \ No newline at end of file diff --git a/onnxruntime/test/testdata/ort_minimal_e2e_test_data/test_voice_commands/test_data_set_0/output_0.pb b/onnxruntime/test/testdata/ort_minimal_e2e_test_data/test_voice_commands/test_data_set_0/output_0.pb new file mode 100644 index 0000000000..cae2717640 --- /dev/null +++ b/onnxruntime/test/testdata/ort_minimal_e2e_test_data/test_voice_commands/test_data_set_0/output_0.pb @@ -0,0 +1 @@ +Bdense_1Jh==?=j= "=#==J=~=` ==#f=B==W|=k?=B=X==c=4_=)=/"==[@=4= \ No newline at end of file diff --git a/tools/ci_build/exclude_unused_ops.py b/tools/ci_build/exclude_unused_ops.py index 0c6fe57857..df8e1636e8 100644 --- a/tools/ci_build/exclude_unused_ops.py +++ b/tools/ci_build/exclude_unused_ops.py @@ -345,7 +345,7 @@ def exclude_unused_ops(models_path, config_path, ort_root=None, use_cuda=True, o log.error('Please specify model_path and/or config_path.') sys.exit(-1) - if not ort_root: + if not ort_root and not output_config_path: log.info('ort_root was not specified. Inferring ONNX Runtime repository root from location of this script.') required_ops = _extract_ops_from_config(config_path, _extract_ops_from_model(models_path, {})) diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml index a57021520d..e495f49a7a 100644 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml @@ -1,5 +1,14 @@ +# This CI has 3 major steps +# 1. Build full ORT, install the full ORT python wheel and use it to generate ort format test models +# and include ops config file for step 3 +# 2. Build minimal ORT including all the kernels and disable exceptions. +# This step is build only to safe-guard the --disable_exceptions option. +# 3. Build minimal ORT include only the kernels using the include ops config file from step 1, +# and the models from /onnxruntime/test/testdata/, run UT, and use onnx_test_runner to +# test the ort format models generated in step 1. +# Exceptions are enabled in this step to help debugging in case of CI failure. jobs: -- job: Linux_CPU_Minimal_Build +- job: Linux_CPU_Minimal_Build_E2E timeoutInMinutes: 120 workspace: clean: all @@ -16,21 +25,30 @@ jobs: command: login addPipelineData: false - task: CmdLine@2 + displayName: Build full onnxruntime and generate ORT format test files inputs: script: | - # We need copy the related test files to a separated folder since the --include_ops_by_model will search the testdata folder recursively - # and include many unnecessary ops, minimal build UT currently uses .ort format models converted from the models we copied below, - # which will be used as the input of --include_ops_by_model to have ops to be included for the minimal build UT. - mkdir -p $HOME/.models - cp $(Build.SourcesDirectory)/onnxruntime/test/testdata/ort_github_issue_4031.onnx $HOME/.models/ - mkdir -p $HOME/.onnx + # Create a folder for all test data + mkdir -p $HOME/.test_data + docker run --rm \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume $HOME/.test_data:/home/onnxruntimedev/.test_data \ + -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 \ + /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh + workingDirectory: $(Build.SourcesDirectory) + - task: CmdLine@2 + displayName: Build minimal onnxruntime and disable exception + inputs: + script: | + # We will try to build the ORT minimal with exception disabled + # Only the building process is verified here, no test will be performed docker run --rm \ - --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ - --volume /data/models:/build/models:ro \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ - --volume $HOME/.models:/home/onnxruntimedev/.models \ -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ @@ -41,10 +59,23 @@ jobs: --skip_submodule_sync \ --build_shared_lib \ --parallel \ + --skip_tests \ --minimal_build \ - --disable_ml_ops \ - --disable_exceptions \ - --include_ops_by_model /home/onnxruntimedev/.models/ + --disable_exceptions + workingDirectory: $(Build.SourcesDirectory) + - task: CmdLine@2 + displayName: Build minimal onnxruntime and run tests, exception enabled + inputs: + script: | + docker run --rm \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume $HOME/.test_data:/home/onnxruntimedev/.test_data \ + -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 \ + /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 displayName: logout diff --git a/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh b/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh new file mode 100644 index 0000000000..6251f6ef74 --- /dev/null +++ b/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# This script will run a full ORT build and use the python package built to generate ort format test files, +# and the exclude ops config file, which will be used in the build_minimal_ort_and_run_tests.sh + +set -e + +# Run a full build of ORT +# Since we need the ORT python package to generate the ORT format files and the include ops config files +# Will not run tests since those are covered by other CIs +/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator Ninja \ + --config Debug \ + --skip_submodule_sync \ + --parallel \ + --build_wheel \ + --skip_tests \ + --enable_pybind \ + --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so + +# Install the ORT python wheel +/opt/python/cp37-cp37m/bin/python3 -m pip install -U /build/Debug/dist/* + +# Copy the test data to a separated folder +cp -Rf /onnxruntime_src/onnxruntime/test/testdata/ort_minimal_e2e_test_data /home/onnxruntimedev/.test_data + +# Convert all the onnx models in the $HOME/.test_data/ort_minimal_e2e_test_data to ort model +# and generate the included ops config file as $HOME/.test_data/ort_minimal_e2e_test_data/required_operators.config +/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/python/convert_onnx_models_to_ort.py \ + /home/onnxruntimedev/.test_data/ort_minimal_e2e_test_data + +# Delete all the .onnx files, because the minimal build tests will not work on onnx files +find /home/onnxruntimedev/.test_data/ort_minimal_e2e_test_data -type f -name "*.onnx" -delete + +# Uninstall the ORT python wheel +/opt/python/cp37-cp37m/bin/python3 -m pip uninstall -y onnxruntime + +# Clear the build +rm -rf /build/Debug diff --git a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh new file mode 100644 index 0000000000..d0c7819c2a --- /dev/null +++ b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# This script will build a ORT minimal using +# 1. The included ops config file generated by build_full_ort_and_create_ort_files.sh, +# 2. The included models copied from /onnxruntime/test/testdata/ +# The build will run the unit tests for the minimal build +# Extra E2E test cases (converted by build_full_ort_and_create_ort_files.sh) will be run by onnx_test_runner + +set -e + +# Clear the previous build +rm -rf /build/Debug + +# We need copy the related test files to a separated folder since the --include_ops_by_model will search the testdata folder recursively +# and include many unnecessary ops, minimal build UT currently uses .ort format models converted from the models we copied below, +# which will be used as the input of --include_ops_by_model to have ops to be included for the minimal build UT. +mkdir -p /home/onnxruntimedev/.test_data/models_to_exclude +cp /onnxruntime_src/onnxruntime/test/testdata/ort_github_issue_4031.onnx /home/onnxruntimedev/.test_data/models_to_exclude + +# Build a minimal build with included ops and models +# then run ORT minimal UTs +/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator Ninja \ + --config Debug \ + --skip_submodule_sync \ + --build_shared_lib \ + --parallel \ + --minimal_build \ + --disable_ml_ops \ + --include_ops_by_model /home/onnxruntimedev/.test_data/models_to_exclude/ \ + --include_ops_by_config /home/onnxruntimedev/.test_data/ort_minimal_e2e_test_data/required_operators.config + +# Run the e2e test cases +/build/Debug/onnx_test_runner /home/onnxruntimedev/.test_data/ort_minimal_e2e_test_data diff --git a/tools/python/convert_onnx_models_to_ort.py b/tools/python/convert_onnx_models_to_ort.py index 2998dd086b..de6c43c4f7 100644 --- a/tools/python/convert_onnx_models_to_ort.py +++ b/tools/python/convert_onnx_models_to_ort.py @@ -24,7 +24,7 @@ def create_config_file(optimized_model_path, config_file_path): def convert(model_path: str): - models = glob.glob(os.path.join(model_path, '*.onnx')) + models = glob.glob(os.path.join(model_path, '**', '*.onnx'), recursive=True) if len(models) == 0: raise ValueError("No .onnx files were found in " + model_path)