From c8cc36ac4e4ea85c5ba31f0b752133122faa5f8e Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Thu, 15 Jul 2021 10:27:46 -0500 Subject: [PATCH] Qualification tool support recognizing decimal operations (#2928) * add decimal check to potential issues Signed-off-by: Thomas Graves * add another test * remove unneeded test and updated readme Signed-off-by: Thomas Graves * Update tests Signed-off-by: Thomas Graves --- tools/README.md | 5 +- .../spark/sql/rapids/tool/AppBase.scala | 18 +++- .../tool/profiling/ApplicationInfo.scala | 46 --------- .../tool/qualification/QualAppInfo.scala | 18 ++-- .../qualification/QualEventProcessor.scala | 4 +- .../complex_dec_expectation.csv | 2 +- .../decimal_part_expectation.csv | 2 + .../decimal_part_eventlog.zstd | Bin 0 -> 25495 bytes .../qualification/QualificationSuite.scala | 87 ++++++++++++++++++ 9 files changed, 120 insertions(+), 62 deletions(-) create mode 100644 tools/src/test/resources/QualificationExpectations/decimal_part_expectation.csv create mode 100644 tools/src/test/resources/spark-events-qualification/decimal_part_eventlog.zstd diff --git a/tools/README.md b/tools/README.md index c389ba1b31f..5abeaa6677c 100644 --- a/tools/README.md +++ b/tools/README.md @@ -120,8 +120,9 @@ outputs this same report to STDOUT. The other file is a CSV file that contains more information and can be used for further post processing. Note, potential problems are reported in the CSV file in a separate column, which is not included in the score. This -currently only includes some UDFs. The tool won't catch all UDFs, and some of the UDFs can be handled with additional steps. -Please refer to [supported_ops.md](../docs/supported_ops.md) for more details on UDF. +currently includes some UDFs and some decimal operations. The tool won't catch all UDFs, and some of the UDFs can be +handled with additional steps. Please refer to [supported_ops.md](../docs/supported_ops.md) for more details on UDF. +For decimals, it tries to recognize decimal operations but it may not catch them all. The CSV output also contains a `Executor CPU Time Percent` column that is not included in the score. This is an estimate at how much time the tasks spent doing processing on the CPU vs waiting on IO. This is not always a good indicator diff --git a/tools/src/main/scala/org/apache/spark/sql/rapids/tool/AppBase.scala b/tools/src/main/scala/org/apache/spark/sql/rapids/tool/AppBase.scala index 2a9166463df..6944d0b5b36 100644 --- a/tools/src/main/scala/org/apache/spark/sql/rapids/tool/AppBase.scala +++ b/tools/src/main/scala/org/apache/spark/sql/rapids/tool/AppBase.scala @@ -117,11 +117,19 @@ abstract class AppBase( } } - protected def findPotentialIssues(desc: String): Option[String] = { - desc match { - case u if u.matches(".*UDF.*") => Some("UDF") - case _ => None - } + // Decimal support on the GPU is limited to less than 18 digits and decimals + // are configured off by default for now. It would be nice to have this + // based off of what plugin supports at some point. + private val decimalKeyWords = Map(".*promote_precision\\(.*" -> "DECIMAL", + ".*decimal\\([0-9]+,[0-9]+\\).*" -> "DECIMAL", + ".*DecimalType\\([0-9]+,[0-9]+\\).*" -> "DECIMAL") + + private val UDFKeywords = Map(".*UDF.*" -> "UDF") + + protected def findPotentialIssues(desc: String): Set[String] = { + val potentialIssuesRegexs = UDFKeywords ++ decimalKeyWords + val issues = potentialIssuesRegexs.filterKeys(desc.matches(_)) + issues.values.toSet } def getPlanMetaWithSchema(planInfo: SparkPlanInfo): Seq[SparkPlanInfo] = { diff --git a/tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/ApplicationInfo.scala b/tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/ApplicationInfo.scala index b53d6a7008e..eb8753d63dc 100644 --- a/tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/ApplicationInfo.scala +++ b/tools/src/main/scala/org/apache/spark/sql/rapids/tool/profiling/ApplicationInfo.scala @@ -953,52 +953,6 @@ class ApplicationInfo( |""".stripMargin } - def qualificationDurationNoMetricsSQL: String = { - s"""select - |first(appName) as `App Name`, - |'$appId' as `App ID`, - |ROUND((sum(sqlQualDuration) * 100) / first(app.duration), 2) as Score, - |concat_ws(",", collect_set(problematic)) as `Potential Problems`, - |sum(sqlQualDuration) as `SQL Dataframe Duration`, - |first(app.duration) as `App Duration`, - |first(app.endDurationEstimated) as `App Duration Estimated` - |from sqlDF_$index sq, appdf_$index app - |where sq.sqlID not in ($sqlIdsForUnsuccessfulJobs) - |""".stripMargin - } - - // only include jobs that are marked as succeeded - def qualificationDurationSQL: String = { - s"""select - |$index as appIndex, - |'$appId' as appID, - |app.appName, - |sq.sqlID, sq.description, - |sq.sqlQualDuration as dfDuration, - |app.duration as appDuration, - |app.endDurationEstimated as appEndDurationEstimated, - |problematic as potentialProblems, - |m.executorCPUTime, - |m.executorRunTime - |from sqlDF_$index sq, appdf_$index app - |left join sqlAggMetricsDF m on $index = m.appIndex and sq.sqlID = m.sqlID - |where sq.sqlID not in ($sqlIdsForUnsuccessfulJobs) - |""".stripMargin - } - - def qualificationDurationSumSQL: String = { - s"""select first(appName) as `App Name`, - |'$appId' as `App ID`, - |ROUND((sum(dfDuration) * 100) / first(appDuration), 2) as Score, - |concat_ws(",", collect_set(potentialProblems)) as `Potential Problems`, - |sum(dfDuration) as `SQL Dataframe Duration`, - |first(appDuration) as `App Duration`, - |round(sum(executorCPUTime)/sum(executorRunTime)*100,2) as `Executor CPU Time Percent`, - |first(appEndDurationEstimated) as `App Duration Estimated` - |from (${qualificationDurationSQL.stripLineEnd}) - |""".stripMargin - } - def profilingDurationSQL: String = { s"""select |$index as appIndex, diff --git a/tools/src/main/scala/org/apache/spark/sql/rapids/tool/qualification/QualAppInfo.scala b/tools/src/main/scala/org/apache/spark/sql/rapids/tool/qualification/QualAppInfo.scala index e7cea6eea96..4ea547abcdc 100644 --- a/tools/src/main/scala/org/apache/spark/sql/rapids/tool/qualification/QualAppInfo.scala +++ b/tools/src/main/scala/org/apache/spark/sql/rapids/tool/qualification/QualAppInfo.scala @@ -55,7 +55,7 @@ class QualAppInfo( val jobIdToSqlID: HashMap[Int, Long] = HashMap.empty[Int, Long] val sqlIDtoJobFailures: HashMap[Long, ArrayBuffer[Int]] = HashMap.empty[Long, ArrayBuffer[Int]] - val problematicSQL: ArrayBuffer[ProblematicSQLCase] = ArrayBuffer[ProblematicSQLCase]() + val sqlIDtoProblematic: HashMap[Long, Set[String]] = HashMap[Long, Set[String]]() // SQL containing any Dataset operation val sqlIDToDataSetCase: HashSet[Long] = HashSet[Long]() @@ -118,6 +118,10 @@ class QualAppInfo( }.values.sum } + private def probNotDataset: HashMap[Long, Set[String]] = { + sqlIDtoProblematic.filterNot { case (sqlID, _) => sqlIDToDataSetCase.contains(sqlID) } + } + // The total task time for all tasks that ran during SQL dataframe // operations. if the SQL contains a dataset, it isn't counted. private def calculateTaskDataframeDuration: Long = { @@ -128,12 +132,12 @@ class QualAppInfo( } private def getPotentialProblems: String = { - problematicSQL.map(_.reason).toSet.mkString(",") + probNotDataset.values.flatten.toSet.mkString(":") } private def getSQLDurationProblematic: Long = { - problematicSQL.map { prob => - sqlDurationTime.getOrElse(prob.sqlID, 0L) + probNotDataset.keys.map { sqlId => + sqlDurationTime.getOrElse(sqlId, 0L) }.sum } @@ -219,8 +223,10 @@ class QualAppInfo( if (isDataSetPlan(node.desc)) { sqlIDToDataSetCase += sqlID } - findPotentialIssues(node.desc).foreach { issues => - problematicSQL += ProblematicSQLCase(sqlID, issues) + val issues = findPotentialIssues(node.desc) + if (issues.nonEmpty) { + val existingIssues = sqlIDtoProblematic.getOrElse(sqlID, Set.empty[String]) + sqlIDtoProblematic(sqlID) = existingIssues ++ issues } } } diff --git a/tools/src/main/scala/org/apache/spark/sql/rapids/tool/qualification/QualEventProcessor.scala b/tools/src/main/scala/org/apache/spark/sql/rapids/tool/qualification/QualEventProcessor.scala index 518720b3272..27d52ce2123 100644 --- a/tools/src/main/scala/org/apache/spark/sql/rapids/tool/qualification/QualEventProcessor.scala +++ b/tools/src/main/scala/org/apache/spark/sql/rapids/tool/qualification/QualEventProcessor.scala @@ -110,8 +110,8 @@ class QualEventProcessor() extends EventProcessorBase { } app.sqlDurationTime += (event.executionId -> 0) } else { - // if start time not there, use 0 for duration - val startTime = sqlInfo.map(_.startTime).getOrElse(0L) + // if start time not there, use event end time so duration is 0 + val startTime = sqlInfo.map(_.startTime).getOrElse(event.time) val sqlDuration = event.time - startTime app.sqlDurationTime += (event.executionId -> sqlDuration) } diff --git a/tools/src/test/resources/QualificationExpectations/complex_dec_expectation.csv b/tools/src/test/resources/QualificationExpectations/complex_dec_expectation.csv index 8a3bda14447..02eff54cad5 100644 --- a/tools/src/test/resources/QualificationExpectations/complex_dec_expectation.csv +++ b/tools/src/test/resources/QualificationExpectations/complex_dec_expectation.csv @@ -1,2 +1,2 @@ App Name,App ID,Score,Potential Problems,SQL DF Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,Read File Format Score,Unsupported Read File Formats and Types -Spark shell,local-1626104300434,1211.93,"",2429,1469,131104,88.35,false,0,"",20,12.5,Parquet[decimal];ORC[map:array:struct:decimal] +Spark shell,local-1626104300434,1211.93,"DECIMAL",2429,1469,131104,88.35,false,160,"",20,12.5,Parquet[decimal];ORC[map:array:struct:decimal] diff --git a/tools/src/test/resources/QualificationExpectations/decimal_part_expectation.csv b/tools/src/test/resources/QualificationExpectations/decimal_part_expectation.csv new file mode 100644 index 00000000000..0507cf03a4d --- /dev/null +++ b/tools/src/test/resources/QualificationExpectations/decimal_part_expectation.csv @@ -0,0 +1,2 @@ +App Name,App ID,Score,Potential Problems,SQL DF Duration,SQL Dataframe Task Duration,App Duration,Executor CPU Time Percent,App Duration Estimated,SQL Duration with Potential Problems,SQL Ids with Failures,Read Score Percent,Read File Format Score,Unsupported Read File Formats and Types +Spark shell,local-1626189209260,1052.3,DECIMAL,1314,1238,106033,57.21,false,1023,"",20,25.0,Parquet[decimal] diff --git a/tools/src/test/resources/spark-events-qualification/decimal_part_eventlog.zstd b/tools/src/test/resources/spark-events-qualification/decimal_part_eventlog.zstd new file mode 100644 index 0000000000000000000000000000000000000000..d963f5af3af2b513a24e67bb833dcc57d213d829 GIT binary patch literal 25495 zcmV)qK$^cOwJ-god=diyB)tNfie7UxkV=ZG28Xn5M4sFPqh{M*@^>hzt4LG7>KvkT zRvyK0TxyzVmaepO6iyRF00000kN^OH?gsD%I|nq*IyDM|eB{}g1_%j>f10(BS{!Ho zrjFGqWlz`$g-Io%K#@S^_7Hn^XM{H24UUkjTy0IW@n65WO;8kRTC+5mv3+PKjNqT? zyxw1$iL>LUZ53W5V9$AK+HGy&p#mWuAd<;vmakKLo2D+`s}ufJZB$UGxHk$FcuIJ+ zoaMd{tCLQT7YP)@;eetBLptVkVkW*2`z2ZlU7NXX_-!}B&sbS!2Gh*z+Wz4wA7<&h zx1VRLIfk+w!%d#CZ$l7*69z_Dt)|)do>ALrd=g@vJ$jVR%agb>sI}+L6o`rAhr#}i zA&m{QOxrWq8T`#^n`&#rcXOWDXe^#!fS5?f8#b?MBXVLazVAhBuj(*kDeaf#t)J7Y z&3`t(OKgkMzT%zkaS?K2a$*qY+)y*$UU*wlh}C&-YY@jepUqptXz_)2UN+1+rozv$ zFX&jS8Ly6E^^46AbL!IJXgDAfT9zla7;Ws?Pik%VeVjPE_MYC`PkMHGH5;F%wdnnn zRMyVm=Q%z8*QdMxU>;2ZPg}x>Q>W- z;}KO5lCt67oU=*k)S!P`5Ie@w*_Y%h_cX0Rt)*XL<6O5#8)G%c1`;d)0zmlh38Cug;gTz0c)FW1Z52T;=|@1~0sP=PYKwpwsKxvTrOL)blL% zdzPe*aq5ui)|W5bdEvu0Uylxqg#>A!L@Y)dLeV8!YQcZUSFcl3t@89AG?`_L7ldIgzw$w3{b+Z_| z#J1qy&0=h#kfI`qA-T%Ag!vHrb!`^BpwoF}_c#LxfMCD@0xA{@>47FB;5mTsc#wqG z4R{bp;lKf|aus{deINGUpi8SgueLtyb#|^cKFwAq#do`4n{lo26+KruO>6Kk-x*rg zB3-tnaD#3BnOWB29#^4AG(12e5)TQeP$U{C6ecQ_23I-Scg8yLg>SK2*e@{7Ts_NM z!<%!)fUjNPCQrP@$2o>wdp5uKnP;y4V%>bd%RcOuwpIANPU_ms&})x2279g(=5{ON z_DpN&ag&a3Yw+IIJpYE`bTU9D%58{Y>ncaDHWli$q!uB~v9+zjf3J=^)33FyN?R9^ ztDFX_ojH1~Ho~4B{^cI$Py6yLc9f1gduZX}lkGUm5PKHu)>Lb#VOVi*+~XeFtZOsF znz1r(*+Y*LgZ*IxLwY=knVHMgHK$i|z%1?D5z+{cS^A^|r)#b3?A2+#D4ObqGYAMc zpdBrBmBXIDwOoDD2(hvBx_sQ04dS*y+B45O;k_*XmSz^4h*hxF0BT!(@lC7pHk1h!h4&sXo^8KERPn~i&K0& z2{EWAIV4{rilWqjtVge7g@!{hlGo(0F6mLz!g`Uhpx%OUSx-U^UGnOyWd&C`JQwnI z^JZ~d;OF=JDiRHp2}lIxLC^ugNP1EPC5lHqh3pvatnk#@tnhN)#!tax-@jm+QHZer zrr?^*6ciLHr}J`^d)$_Hg-Su8s@%jI{BIf|y^bG~!!fz*8_uqB(y7(eA!Z+Pb&O{a zUU1)cGh@FNPgFQ4f>Hz}2SYnj+~W{PIyVK5=f+Gtalnxt5(NYuFltO=ivq=vTEC05 zRKNB)USry#pkXWu)}#vfHwD|^G^Q1B6LM=9f`J0=ah_Y=1-ZIbEuNSPUDs{Ts>B2> z9+9+|rs(ltL=cX)X;ae>?x|Qz4Qi@f#&%*tDtEQc z@7k#j|R#s==9_K2T@5GVfY_MZ&{JG$s zsB*b(R)e^wa;|lRV;kLs%6)#{BKX9?HgC~268AurlMasK9w%K&h|_slJmC><_I(0? z3Kzi$POfs*&d1Kcd0fVwDVTl0UOoSOxZu7NSGimrBXJV~eNiY4@G~f3B;4TH1xR4C z48&4Sj2&a+%$=VBZa2c)+Cu~av1!o$+b%X;u>Y2=#hTXegd2YLiakdl69wj}X+1r3 z&9nH}^VP88&d}L;qO#5V@;`gkrWa}zJ60!!@7}JQ_Oo^tU4s-hQ)GGTiU|f9_l38$ z%EJN(gJj{F*fEg)gM>ZT%|-|#u5wx9-)(u1AEo`OXPKA%tn-4FwNSs|{B=UF+Gj}% zqvwbwy_k0)^t#~-Cyf#ol&f647)q{kPjd}3QQ6q74O-R$|85s+>C0I9UuA@Ys~l!2 zgR7jZooAgFj}Xr-f@4U(^E%n#wqJ*8_K z28zTjYiA0~UgO0NDK~z3^Q-@+fN|*(YF5_9mgbtRxHIq>unHx(sZ+Zo{A+HRn;nCG z_1oGPY%>*q$!zf+6^P&*d56&d>R0003UQ9`z2W=a93~Qq=h@iA#+<&!w1wjFL-APJ zm}v|jyR^~q_Kkg^X>DV@{FA=TXUtLv^VQt(eYC5cdCF>I-5_>IQJxlr8^2IGFWE1# z@0*@<+4x10tK5?ldycXH*1x@p^HWac^lIhH54CIy*aVcnlND0^R~#<+7*Q{9BYJ73ttpsb_G{CtMkS$7ykJ2JC?Ykq%#9$oz~+g{o8tfTYKn^^ZP4We4|Sn zaXPQpw06yG>yyo%BW*;JyLNH4^s}dUcw}J-fSYw-?VmKa>sJ-w#`Y$!4o!IZ|EQ1}5!@nxl z7h;>`ts5!{L`WVeBnu=96wtV@W~roU4brp*JGX?|rp@n%+NRAqwLClXusR=B=Y1D! zGa9y6ZhGIn{pgvucljxwrnPtZ`83xq{I^$~58Eu4n5A`U5&|*Nu*O=l_G?_$`Q*H} z-$g=1!~Py)F-7@3^M1n|%j)p+L#Gx)>Du<+Z_Ay3f}kjZkEPTbB+LVa@A%j#GWY!{qp8WpSglUXvAx`auue^S=aw%|&7oy2j0Z$%T9cDD2EMD> z$e{kRAhP^Y(e`Du-G6Y<~A?n2F~6uuH!f5UWm~Z@xu%n%;bipw*Wm91?JtNJt}?I&2`LGR9wj1CIxoji+v>xvpb#y^_TQt$*T#%m zdn+7IH0P{kJD$;BHdx+(A~VY~ziscj{N;B`gT!}GI`6}bZOfR!l~=o|T}g^<##N5S zenbveIZbQ%ni&fGe9g?mzdDP!{ow{rDQQ~E(dp%rIIkJ~QY^+mpv9PW9?HZGAa)6p z_Qa8^oTN5;$63C7XSzkd4=roC)Gdq9&K&te?AQHbtpwPaX7L5z?O`h0JeBT2kLc*akA@M{=KtM=Do&`w-5|KFN08LtpKT}T<93^G135&QLbe8H{3>>pxv#_AW8lcu#evDU5$LauTb+#2@WnU}G2 z;}EY_1yqk-Rx9B&dJj<@I(_?P8)lhE+5|I6y1Se=ww{5F1h|FkMW z(8Dp4_Pg$vht>JyPg$R<+y*zcCWX?S5-b>9PCF%tJA;44i=Y1nvFG0c7Rj+@(b8HT zrAhH$2riz+=9kuHB1pmMNornf z1fvKpVDw(ZwxNB$+HPrNomS_Tbmk4R?QR{H z+F$Mb>RIFSHkO*ViQpt@I)4L37V4t^EbNzG;A|8)u6kQ*L+NdD_L{L6H^hTrgR1r| zk7uDam#X%$j&JkUla7IzIza5tJZEMXxcvVGyM=ixZUI+xTOD!03H$dSx;D z6#p39WEeJ->W9><4U)ztX8gPQrbIi;pREd-2zV4}bn0148KId^;-gBJi)Y%Zv zNr0#H4d(TvD)<%JiX$S@mMPa*+vCYCZ3u+k6R_8-y0q{bt z%`j&=fhbx4%$bID`;RlbjWL)rZCM2-C=;-5k3Hv1Yc@hl@dm4JSZ%sh3uQsB+izL4 z51OI}!9@fqS2@T+1f)Wth*&^zm8;uhd$El;PGe&Ui*=k?M4{uvu0bTYHuGbsEO3>> zmnHk-&SLa`OU35*O0b#eaiMJHE6~a3;D@bmnUXD z=^?>|$mQc(~gC+4&l(=GM)@HO!PTV>MB=z%IbCdiUWj#1r&+}CXfk=d)!AH z>DXt*$y33j7*4jAl5NARuEDy!dtBunS08wt)IH8U6&0@8Y{P#gCKLvzGuPfCz<9BH zoJ75ggp941y5;IvH=yz)5CM638Y+;9hzW!N4G~DBLUCB2Scs5JNG1~$4Hd`)g#vkq zJQIfn3Wf-Y2!&yxfnq{JVgb_dNFXE{C?E_V3QX#GW|@bnZdr>oX-ptuG7&)jM68X1q1I;L-JG*Yqk;kiBa8y0K}`w@)y+9`m4g%5UW+GS zjFZ}jLlSxT2AmrIuG4argZXD>VaW1qfI%dnnRS8mZw0d|kT3dqTGm4UAqdgVHbc>( z&YYjN)y1`uI;D;KIv__?@gh#=<#aMs92JVh1oqtV_3G55XRtbD^$Rn$R>S;QNY%6B z=NoOzqh06q?rp}Sw6VQ5xbe&PYp&G^9mDb;b|}Iu#a=fC%Nt<-1##xi6lhHeb3yXp z0AcuxF+m7g+Vc!r)`E$SVMChMpuKQ6xXL|zpSLya_q|S9C#{})b7RhUUjDZB77)@D zEv0gPkGnolWA}=#f*=I7#!UG0+4^Yg=eRQxclL7h>=ND!nLrcuh!!Ri5y>R;9;?&k zZ4F9`uj30_^KUDQ-djW$2@6n+1*Dy=KgH#z)apyXUovx^1XSFcihh4=DBZPw`E4dA z#OiD`B8i$J|7!0$hNOGFTlS|kk%@`T7Ee^%48SnnDe>g%F+v`{o65`8bG4Bt%d^3g z{uba-;}1qiI8-8%hzN-Y1*8I5prDX2aHvEe6?a~~5&^SUDj|@{hM`&DKsg z`C*p(N}Q`lSGlLvII&m9sOW*9>46x2iJRr1s7swx+dTa1N#EeswCyKotbQdLyA0HG zb-?Z=U6V?82$3{Al!_#Qip^*9dug-m+I7D?G_6TcR8=6TfF>kxts^1?0`dnCVE=6Z z2kz~swd4Gmw=}}PD)5Y=`J3!M$NDX;h(H#ehz1f3 z5mz}nt-c+9;|oatR^W6pK>Gh1zd-uGpj6Y)e^Aaqrz~5TfG0!H!QxiPzd47W1o;~mMYW)ee8xvd>->kAG&DF7f zw+ahoccOBuauF663LWKwR4IKOgV9(tB#CM)7Ssey56K}lB!?V1f$(TNk13iK4r+=TjYygnQB*;U zX^I++DQYYjjc9r-8jWb0q-l~C42WVl9uH|TO%FK!~0N<-kGJztE5t(jvQOAxR6Tb4$k5a7>fs*i1##L`f7yH7Ew< zpcYgk09FK{@sQkcNEHyE&sglZggLo{`4U#AHZ$eaj{j6RZTo1doOHl8w^Lo^COB}F zbB}YCbB}}Iz*WvYF8db2JWj>k46Bj$PqOn#$pkGb|?@B0d)WhK+U%PbSfu;U^-AA z)CUkGr*hb9IKs0jpPxaK0pv1aGctaA{}rtc48~IBhGZdaS(7G0UW2Q4-ilX zoE+U^@q~kb^Q!s)0Tdl@a)!+b_F5gBI;{fPcSHV;vMpBNAW$D9NZ{0_6V8N_Ai+Xy zI?-q}8jc3lkf;VkH5?5_gK9`q1DYC)2IC=Re}Cy7$8hGPU( zLACR?H&h7;2nGt=_67(B5D=z`Aw^OINtl`*6SbhI34)}$O(Y7ED2VZBY&Q};74JB? zZZ_f`-*B!R?Yhxs;GTses@&ryL<~oT+bHhaCyuO=u2ibz@?D{G0S5O#nQreBCjkmY z5|NlpI3VgFU(F#0_dpXa-xbMGxtuH914+VG=-0-FQep^Bt)`o{#RzeaL$IJ46+A#` zM@SnZk?j)$bvA?p#GW1B_!WMAfWYxMfYqk;0fLD-fYk+wQ8lJ52b=?tUo56hNU!FA zEd&DA?dt;s4A0xuY}`no26CPF8h{39W1Pl?frxrE7E?4m8V3%LXPpLsnQ6d)Y{v#5 z6?vW1Kz)FK0XeDzP;1Q4xfdXf@Q}{VzTh4R5M089A`{S4aiKWwfuv`#BN(>=_f$j^ z>F*1byZl^m{1>*^jn?m^@pRPKSC?!g4R zurT?Id*U|4!otGDJyFHEtm7UxN!05kwt$O`T1;DB!?vOT^{lkKdHXkw zF~l+44+=CQsbMSaW{0L6db^lTPeyYzxb-lL1BxN?ST+Kxk^! zuhq=Uh0v?^EfTb6r3GplBUS5|Vqlo1QC>FpcKul`Pv)1ffF}GFZ)ut&2)!FEDLp%L zW3?$Hfd|3G&;Q1><$0VNCS9^gV{~chW}(+H=JKFu!f$zgH@S05i!qJWncLc8oe!cs zHqH)+TTW9S@1T{-ysXF#t+DJDR_6r56mot{KF#AO{M8+G>gTvu|MhK#JDBQ%(Ylb(7g9Mqhkh?E?Lz! zYU2c_m-%fiPX$<=xb0-ha%(T+*8V-V5nF|;+L)tT>V%rcv}L*^(APL?{8i8HILlhs z#(nFQ*k&=ewSdt3=dt>fTTAT!@N3%90i#(hmV6Q0D1y+GA`69CDxp9m50VLlpaVkh zMz7YQn&#ziwjwwVV7!5)I(+~gvFeDg^h=ub( zjaeTQfB{fmP=HKIFqVfx(O@74fgA*36a+yCA%-Ag2tvjfxCQ`;t9(NTqKq*$=@MNb zubJi#l&|tZ>8?@?i$=EQX25t$9PEBOhd<0hw44bFDwU68Rxv;4+?Z#tO%a*l@&Qp} zy1YHTt=wdiX`mWBvo}|09As?S05Ir$>OAo)W{>u}^!l9tXFCN#0)%7!fUkSJ^c_hF z4b|o(CVo%4vaNC!{YY+$Z_s^nibVHUzatT z?TrbCX`w=m_uYdJ=^(H>7LKeylSyA3Bk1`$A{Dlyk4fYz_ zg;kaA&kPDjFSBA5+(57O?tus^Ds+Mf$?La&Q>_=>4>UKR5C6XMt~G1P@3#N1R3Gy{ z=x+~5#q-!|IQCeq%|8YPtOt6yVO;ZnqX5q=kNk6`{qJCMLjikI`SZ zvQ~C~@>Lw@aXIwL1Dj=}Gzpkn1f*l8150LlI{lm;k5(S&G-;Z9CeVk^6{hR#?*>a4 zFEt@rJ#419EI7;Eg0^BFhqa7<%>{DN#fNfv@i!Rl--%$9xY>h(`J5k|#PERBXgfa^ z5F>-3pY*Fp#?CMWzF~$BZT=b7NPtXddPG=qj30zV69qZi zppK9t(GidZ8maZk|D;5`&_@h7X{O4F)7mg3f4P zwt+rKFTL_eM18+rENr=Ur#WD~Nlg)u4Ki%MN@xf~vPu>d+W!GIndM&-6Vyp1#4~J0 z_d2XFo;mOAKuKpVKjmeq^rQ1KpsP>Tq$0T5-EsoVG6<`H#4PA5tD_*R&O@hXlf#Dl zC-GA0{N%|5907}ZfF+#dx+a9eulGjr%5J=O^xi;>F4j7S*dCdA|80O~5}@>XmD@jh z*~*7&6pId$zY@(L`I7MlU)#wf?~Vf%uqq_UeswYdKA7V zpjFn+h9@9u!o}`L_jClcc%w`PV2hmrBOtSP2Wk+K3KJp)#i)>d0>fGC{}d6BxS=Uo z8cz=q=L}3N%5#+=7z9WyMv>iE=Z$Lg>LkQdnULXI@${{D|I7_1m;$$|rWjpWHZ_`V ztPOsf70(5GmVO0GU6rFRS-CS*M7KfZl5+YNsJVx%W#<>B@U|S2G487IeF1LUI^hEIBzHcv0!iT zjw`z!@w=KX2h{SzC)Yl}S8x-qzT0*)#rqK{P@QU4FnU)#tK zKpTnj90$L>Q6=7;!Nmu}C2dj})#l;^>(0%6glHCZk+Z^6G3LL4_%DeAjnbCmD{t)5 zn##OeZW5$44jn*ma)psx<&leSwTt4dQpB055dRI|X)g_EIXLBjUQJ?B(s?ud^v7Q>~D zI{<=mW^Q!QOvIf`a6)jQ3~9o6j3XIcN4EAj#{1u8q}Q>8S&-7K|C!-z-mbc<+UX2E zE3Q+1w6u#eI0e5r;)u)~8vVeY?JMMCB8HZ``;YYT>9%b(vnD57&_R2&!F=nSxo@tt zWpqXjEa}~5Q)IiihW8XE46;Q;P2knAttt)S7aCF37=(0%gPJoEg6*lc3GXfxix{u2 zs*Kl8kvX6idyQi4u&cc{RvL_&-c56K^-pOgKO)pHiTC-cIwAjL>RZC7r`Cj#G7rs4 znquMO6-JdD-&ml@QD{~Y^|r7#-j1K4 zC;G-zLGy<67=EVK>BxJli!B1l%2OQV@|dAfdJ<*|&bt5{*)ADJ0yNf8SGPNDv|doc zawQkTPEJ$t*=Z@qC+~FxgPD$9jo6JlffeKrSF9+;hP+43KU`UCbAee$WWClYIz zyd=s(m+Wt$>YKBa(u>chcFZsmJbWlp1V?b_S>rH%jA$(rT3KXHVCj@x={yGL-roY8 zKwu+~8UISdN}RhYD&t^@R^~$I*tIo;NnMX->e8=c>Vh8dqCkjSzbyR6MlSvLA zg(fq=f%|$<@mK*Y5^5OC!H=O|C33<1MB7H3*x7@*<>C+I7Nj-ykCAimLtO0W`sOkT z%H*R!;JtewL+=6b$>y&ND18_Fq6MJ^v9h-}OXcuECjVh;ukZ;VRD0T9<5v0=uqq|G zaXWFwPc`(!4jeZN7q-4S9v2e6@TA+Z#`q!(4}_DXCagywT7Em7h-Oc3pzjaI2K_p!)$s*DQE%5K5KSKJbb|2b&OrL}fxr7^{|nKd zIMh}`9yEUj5^aCrTOqFtCdy%tbAl%4=T-lNJy@~B2273ly@Ye=vSXKrX^o1Ma7YWY zHfw~##-a4yLTUG20L#30^r>cG<3ncFfZU(i{UURdirnmcK8FOXH_7Oh$SoXiP!mm& zF*Z66Q6vCSL#vX+1j2%lyCF8H1m)mw$D@ktZCe>IpMzHP;Dy<|e6O|!Xhv%p(PTiv zI%Z6e8v*`EtCC;weg)&;1S@UvPWaQP9S7tw;0-61e@Et?JX4*N?Gz_YANXrG(L+LP z@V+cmEphm_XiC+=4!2xI8}^H;6s4ez#i!K#`KlPWz$ALpyc>!yRRm)_6wWYc)3h?r z!cYAl2j?a;H+m4m#Z0n)0^T&y%xCjw?MJDg20TJYC*Jl|w=-`oT%#KIV?2@u!Em?* zd<|6MbP|R{m_0_NR4pWCsZkm9DY_!p$Ja8QFJOhb%?DW#kZvje z-^c{V#n?Dvr13=!@xvorpvM(P<6|>t^(rOVX@a8!npx3XYeCs=;6=F?4dKTwY=7i1 z*ti`Zl6VVvDXAor>AFNq04Bu4pWf#h?L`WtpoH|l1Zezq=X^3E?#DSw4l!Yn$1UWw6L%`WPImtm4 zMd7z+gjlsGx!sMe^$*vz%4^ojod&)oS15M6tWf&=0}Zo&ZVq4(>%(BL08o?eAf6$c z+Y3-?ubB@%V6W(zP8m;D?h7biocZ*p4&4V%!@o{sv9sF3DG%M|Ppj=Nd^Mo1Os)_v zObvYu`|gg6QZ|9g@?Yh2SJ6tS0x=X=0EOZRsu9IrskLg(^W^3b=-| zKpiUgiRKXpf2M?DWAH#IQsD@fDg+~sP)Xs*aPi6VuVjDdOFk_c*NgkT7D<$@kO5U) zx7yiRQDkM0OF})@+qZZyvf~<7J-O}Wy9{D)jbNnxD)pL?Ve;Rm72L~i!Csb%&kkdr zi#u^EIB8SPAkCZatD4QT7cB-&YL2g%*O|@q#@U7tU-GHK_oMTrIp^Dz1t3c8oxPw^ zj(7V<4D8_;6yrh@+Z<|>YKIX_Vr@%x40ABf$@oMBR;uxAV;6e^cNYvrxPnQV&%Qk|@B0x6859c_U_M zakXHb#t}!J2Sdn)+XrK28)ctsM4Ac1y?2hEJH0f;nCNnwlq9%>vte&RkeQgPLtVA7 zzw<>H0W)Vjr(KF~B(+z#N>sW?>N!I*vB${;{Nz)X5srU!Yqrten+2jIgVv!;SRuAt z=*`XFH~zLeJ4bLvXsoJ9?^-it__ca~-aZN0HUeja3b}1@sb+AC`ck{W7gDQ(RDO8; zkXxK1)SDo*ORBp%O-Zri&CFCclsJ((P~1||fnxSs%F^V0)}W(`79g>b3xXlqSie3j&>@; zXx)s9s`L4_ouxZAzw6U>>$B3ANyZl~(2CA120I0GzK6y|X+R@o;GY_nb8NU?ar;Cw zpoz+Fv+H>18AE=&z^29K$Pz-!W=LI!*cast zJcYTw+oR@+ z)TGi}d#W`~aKamUlCpR74{1u)i+1={m<=_7;!0p|It7M@OJB+il_T?1rbOVMX-D>i z(mR~hQj3T{WFs|=(wC=DLtVfZ%{sBpA?FTf>2FM2eAn7hHo0m>n7$$UUDW05#18t_=qWNGkOdR=Tb&q7=@z7b6ef~g7*jppESgd+*6 zSnqwMbbb7}px-02C)hu3-4M>jjs%{J89|Z7))6#mUa6;#1r$&9qoMOencMRm&^b=` zyXQDAI)|eD9$gF=<$OUhBOC|l#E7O&zCnKI^>Fxi)4-!%0aPFg<7GNEDwsZW4;6V# zD_5m*6yTQ2msYBVXT`hz(aZ8UCN^3xMmIkZ}&3gT5lai}UNDl4MCaN!_OVSQmh4S6q_ zXoI9zDcIgvK^88xF)RZKHXx{RKs6ymJ#mpfaS>K`zx4dL;`j(r&E^di#*MxIqPa}b zd5^|_jt!D10g*fah%Vx=cc54%pWOep=eeNO*8jz3&7wZaaKBzb{{_R3-or1Eu)8t& z?=uAn3vm|Uk)1!%Qw+@E#Zquxy1=R$0QH{mBDx_{tebSE$}BBFo~G18|D4d_66E@v zh~F*2?L7E5t(pB{e0{4likREsV5}eeNPqd*^9+<5BF7VvenLPa{6o!M8*CvxyNP;T z2@0+WKbj9k$Mv=G2bUEplUo%@9BHnTp7sHIPd}5am~)!sQu7HG)JL4M3yZ^|aDLQN zUMAN~g7>gig|`%0)!8aZFLAwZuTHTPWN;PSU!^BPSCKg`DsrwTtVkaN~&89cL+kJr+a<+ zh?qgdYZ(`VFs#ESV2#VSGcXVbY5o7*#l7)a`itaw*tdkXyn`%XmF7qZV(b?>pFKvE zIZOv;ob9?(EK%*-92Q8&9R}f#2@E9a8Q-&Y-G|FGWyfW#iWv;T0AuAh-MbSsrph+) zqpYCvdqs}dc>2j+9R3D)Ia&WspQp{*eUP#y6;l!vvLZnd^|Klz1UgWYmN_Z%2dy4$ zXb*{tTlz%sJ;aFAG;3Vnsc5VPC_9i}_k^{$?DHg3NI@_p$pS}WOwD`B&x~4JjY?TX zWLVz>qjDp`U|Wl!@#+89OlZO&fJ(V>MC+5B_}zu%0!GVdhe33_Kesj;+bdJBiY=Fm zv{^LBqew2a7O{xJ0GTqynRj_eY!>E%thkM~D*R;T=6# zI2P~Btr6fltC|*Dt#;MNm$}xS2VoOX@nNz1Fs3ybvkoapUW0g8frX-1-i8l?QqWB5 zAT{*$R(1&5r)N#tcdaSS1qolw=4z2>dS434P}jM@_xP0NKHw5+5b8->+_t#160ZJ5 z7~pS<%>^o_Q_32|OYz%qtVqf|jq1ItyJ(FFdJ6Xe&xykoiaHF=CT!>#Q=rJl?kr=grp&U_UAxCDXGQH`l*JJres(9ydZ&ih#_Zbo4 zFFl-KBfL1R=6Hq{6?~qs2x~*>u4ZNkG;0;1taa1@&^lPX0DKHeULnUUkrw2^SaeN* zU#MVE9}XbAl)&Z+O+NbmZH`}rUYurq(Z5^zva6S%M~X!#)PEzZbNS$^nZ}TAG^0qS zT+I;=pM@`05DbMG_+kR2AkUnnfGSYI7eikSW{pCSD5|tgB|bI<2FTYq#gqv!lP--n zG>t^19~vEzvn|a{p*97@fsx=-;k9EZiqU(YSN` zjpHRda_)q(#d*b{v66r>BM8(xXCfiv?t~j|ARF$+Y~>X3B?Td`mKD(jnb;tL7VnP= zh8k7E5AoT za+tK?;hf4?w)p}^mij?QRNdTC#Mq(Sdm`eeD@|X185oiJ=m7;1sruR#H&Z2n_fVz# z@R$Yu;+j=arr->&{_$?l3o%=-S3O)deSZK2!<4Dw#Ytw_UtRe?avrAB#=>}r_Vy!A zowd2W?MpY?&r(LCT?_~)KBj(Zj)0-qNT9kUh!((^sh<&8v}8`swOd`Ti&X33113m3 z_$mG^?ZjXj8Z_~m4N{Dri{v6B)q@0sb$8x9JEKSnmp3mi3+}ONpqp6f5C1Z7;Ft}M zHTca8wLNaSv$N>T$7p2-Q#{J`t7ZTd_2V2mxotR2lz@(PVU1j{++JtiE~Cq*95Jes z9tG2-zH_g;7zK*&67$XO6}y=RL4z%U6?I$L#1>j1K9mzN=j2E*{&4aiQh@Z!tC!Y* zKHxvT?xbH&XV%9|C}g4S{t>~^jtvso2}5FqveY4Mq5zzPb0B-pREP(>bY)7!dmvBJ zDMN>ME`&J|qoEux+noG=JEJ(8?@P+yh!8MLmjWkyoPwAKaoa@*2b!ro4Um;}e|`$s z+YT9$)>Aq4cu$b8*h^sXxKyticID;0c%6^uh_J5@1)-bep25fZ}!yd#Jjc>6IO^~N3>p(-Kz>a$6e7OB3kcBW8o9X zjUmY{71zkabk#1_;v1Jv34?FEnwqd@*oCRUNej+9Rb+qav>i1;?zmhqCN^YSgN`R5 znpVm@9KZF-Wg&|4=#z;DbkDa1{4 z8}U4oPCys{Jf)STz|H*ZvT&dCyAgmoy$1#rcQ6H%1gA*G+~E0-4r6x=%}7#grl#DE3o{BwwbsNwl>`m}p~O$rzmd z5H?F-th(!{Rc+zzDxG;;77s9=`O}~1FZG>dfWa2vV$q6evr#vVHy}h5V|WNT@={Xt zOerMlz)b#)}s z^aW~0r$&a6NNdp^h`G}eoUz)$ai1jYPEx9$VgHL$`qohgv?$_M=#JrXP|2CR5a>*+ z8f|flK4N=uWuGHzUICIM_Xg-M1U0BeRe(wqq&QDqC6yHSAMHX-X0Qy2)oky(k6Xze6e>1>f@2~S!(X+7vE>wRY{Zn&j!0sp z2B#P>0e=rqP;CN9>sZ`|RR>F6;ymm%xfylU0RF&^P==}dT9{Nv57rKd1Mg@g3oKZd zq4J<66SILh)oDGrRLLla3sPW3X+u=!$^D8s`bF9k>vEDH8LeO`RF!B}$2LJ5 zHR>TfC@H`#Cl`|Nbu-J(&>s)%%Mg{)A3Pr&b@#kz!z_X!08ZS%fpoJKfQZmIBp5c2 zrABe_IVQ4sOqQBvg56XmBx#jKe%23fQVbZKj$sx`3?{dkmqts$=6||5hMeS~gha!x z>}EtwtsuyLL2?LsGwSE=eiBD(`=-OU8VGy+lZDs|UDt3P%$T5ynZz@a4R$%nYj8k@N#;XA-5 z&(f}|wHm~5!O7YXi?}^yQ!YTEO<5@aF@uTKp^?!SICQcy`gNmeJ`3Q%Nzk89()nAs za{&YksSDJqkKR8J_E?N&D`TQ%E0Buugh9bo41ZO?VfwWtJ|sj$$ev#{&ECg}Z-Uc; zmS;Iu$p3_qo8bP4!~po-c{84B)Zu4(GZ9E>07`D$aifYc1~WmfqJSeCrsm{RO5~$M zB|}GV@px)kQZx%fxa4)-4r}8oznWW0@+Yw`;MwD@UWx`3MwXx23~+pUydpFQ?w7w^ zciwR=CO3$Wk|w`O#TTWChjyxEl~Q@NOsud@I2y;={^R*##uFfyLjy+Hu6*mVKFSg@ zwk;Yh5o^!a!97FAC%;Xb0R_}PbNf73;7}| z(XdecQ6P|+Bl4HNt+9d*z!WcwORD>3cf;3Ur|vq#0wBgKjK-kKc~Ue|cK|tlM#fGk zBHea+s0tKD8yq7DACUVzozTE^STKhrr6=cyE2mqAn^q0f*8<4fYq32oLRN;1xMjqI zz7($z)99u{XQcG!8CmMPKZgT01b9J>l6jHy8<~v=o0GM&ZcHo0w5Ol;5ATX( z&*=(@OMOTl>eCAttFJbw3<=2WNoxYN?OTubo3)(&3qQEn1YdvPf>04helg5=QbNFM zL-=f`vPR*=TM7(Okjn;{*jko_XmhFj5*%z0_(4TOE+yMqML`4#f`*h04$mDRd*Cw} zr}KbcjQBg^h5~Q0NJ)86kFr$2opS+Mk`V^pb`jA;O(6W3`b61GfB}!(+3<5RX)k*_ zGM%@;M>APn0Wu@a+)cTUQQ4mbj>~lar0nZF3Qoy}b12`=pOEsQ<=RJKW%ZyKP58~4 zYyqZ6E!sE-r+6 zG_?E@u0N_+3i2fCHbPe(j`~%Fk2jrR7a<-2at=S@II8zp&$pWxk)I!QQi4&c z+{aom$&{SCdC`QrX{ZA}$ATTqECZS@(9v;yO{~!4)6k!2S9GWm@o3MD=n(W<8DB_G zO`1&JJxXrQ)_yfem{PU9U!lyUj7?p#)kn|^((TmvutP0yP7=v|scbap%Lq9SjJS+R zT`+Q$^%;!eD=4r`OyR0z=r@VN&#@|NDg~|N(cyZX>~(%Y1}I9UG~p=;`U{0zv7nqG!~mr|~K|)(A^Y znb?&CAVL`Zke8F?hOWq?wZ)-&#Nd1+LgO3wUmY-1|H6<5e&{BL(YEV0aj1X;@$rcc zZp94GZg&{P$V;R^Ow1sfFv0*cW6pq7gjGR<=KLZ!Tjq0f-jW)6QnD`2P1h7-{O@eC zBYeg;O!dB^z$!5&$Y-$aMqt)S8+TJ=)Gu)Gqg##(7$A1H;~KtQ@! zdGAOc$tfkM3H!h1o>zLuc}dZ--nNA8)fLrHR#NFQ`x3x}N;X81R_ky)K$pJ7=3;c@ zOfU0}SFV$j48z663`P9xYKr9D>CgPXse{Jp;=|JY)!KU@Bus*h4)JiSAOJEDLi3~y zp|8t!!|-KEC7RLr4;>VaGg1w+u~2{kaH<_PlSy!?)$C#2C^S(6c>LuGNxBnDFS@nXEsRkNP%W9iB)qlg2wi z3~eUk(wV_WfXynTfgPAxV5|_&oyt*PpXvpDTMr^>ruUKcmP7lb_M51t-|fqj2&4&(`3a|W_1N1TY10(jJr;3Q zIOG)JA@hAqkJJNsG3m~MVhtan{d4x-Bj6{)&cwgSSnu=U?H{OMfw|rizavTCQ4XKA z%a4#ka!Ys<8l9j~#x}&3R|0Wu%^C@~=LVlq1?(f{;TgVYFFu?PJ4_hlVeUcuKgyV+ zPnrcuDx9t}p(0R7&eIOCkr@O8K#)?w4(VNcdjZc8-d*(X6c4k?1z`zQGRlf`B?}nvBWP;Z_LEoLK zzLP0znHt`aI^;)&21>hwg>{TesZugS83xdh;?W+xwvKI%*zinZj0;f^6rnb$GB$p=6=9^HhrF-I26q_wY`QrMxHIIw8ex z9Nk-g>}!LDG)4=9_&?!6k}T28-GxruG~9%@OhMJIO7Z+HO@x*YBydSLm-L7+=9g)eR?qoC7(_7^v$b`Cw8bd2X3hGi|d^0{$LNl&9g+yslxV+i$p=LKK} zk&(>m z0#rLqVgEuAy3|q+!*J$jp9khlbU&C(lPgbMEh$F#FhN*YFiLvHjxLtiiM=LzL&oSI zA^~iS?ts0WvD8*4JXraMgef^~ly9->n_vngI7y}L)|kMS_h4zAZ^f}@y}oE26uwLy zfMuv};5-ND3rUmh|_qde0I-LIs>V~TA6TX@g5fFAco@V-L8 zK7bZ^4Goi4Dnw#wbRwY9cB;hAjKs%u={hdcz8dD5OEu{2=w{eVjUmN&H`I*wCMQe< zZz>X(@upqpA<+de+*)9hFffoS08NlS@4(*g@PfZMtPZD6=R&_Q%^Jj$Ppmm|8j#|} zgG`!3o>b9w5irja@jy)_E((IA2#6Q=Sg+q0Y%0o$q852qk12%S&>^b5bWEdzO)4}$ zt);x3z%j+sA&f36L5^F=@MdK)WwxzT>E4M-Mc$MM)1>yA-(KECm!wG7AlQz5PRo^0 zcmvtV=UY$~ZB}k!A3v}5N3hx_vA4M|BlvZ$2=ImM!kLfC-?1Ua^RwT!@I$kvNc6}< z1bB9$hEcFIwTTPslRYmL7ZRHR)B$An%qy@fzEjEeOl>$jOP>DT@EwI%e#5ihn^a$G z+T-?E5hi$3NGWsr4*sPC=A)<4_CYCBlZ z0(JtX2l-kl6>x*Fcrk1{q*sDz^P}_)(#r&knSwm#UU3nBb8+0CyklakEQW>zyhITZ zc|mjHasW~e{K-(k93JuXOJWvTpsOGM-eXzxX$J~$!hLVHBji+%@c8BsGfk`l;AjWPhNSSeeW&zYW(z5V(zO)fU?LAIB zN7JIDi1yUeF|lNLk%3EtF>fz?PzA6>g*-m}9Md)?xQbbreRn#XfEYvKP?@?OonfH$ zjTbQhwMbccFE~v)^Gx=1u6+0n2dMY8IlgL4mX86ofuZB;L98PT)X{H!M5@eEymAr@ z>K?}A@cb4l-ycW3T9ABWHCR%cuCF4KPH!MbPDKJS7IaVuy+{@aA}6_i$Ll0LtSG-b zNEgUOd7MkY+vX*-zAy?q&cvb+h|(bl@8T--b#xJb*U{tqIHh?AYG2CBELrRjz0cwGkDmy_muu7tQbK%_uxt9RJA-oUS5mjW+^| zjqn#`TsV1h)ry7j@IpS4HM4%xqZYP=+cAF&q^4)sK4vmc8cquBA2rY?RN1Um{j#2^ z^)+B(y&pm(>ZMn_TM21^3ZUldUgRt%6zL>p6mW$6^lGd7LWtq?o}I6E#hqat39Dxy zek`{C90iy0!7Jfn6rzQ9Q#jl;K_A8iR`k*#iDrLZgn_kCF|IrTD`8;l2STRYPNgN+ z9Y=%GEn8YC$lwHch&9{s1gok=Fw*w|6fH@OZRUhHQavk-55n|+Gy%|CAul<8S;%Bx z{A!Ep5|FN<+fUXjm)nzykZBi^7x2&05qe0fA_YvqzI7Z%DE*5{LP*j7--c&LkvUH{ z&2W%R(^?KfcPoisf9#IMZy(wU81v4=(Bqt%PRP`v0;eq3sxlyiDzX3KyiHfV` z>q3NIfxVRr=*jTG?0@Sv;XxayH^EOG+0mYkg~YKmxa9XvP{b0qzPHiLJ&E6ZKghF= zMV=`E6ZQV=BnoYZA_~$fwGT`4WK7Ao+~2Oe|4Ic;+AUa4Vwnt(EdwD+M-pMe;O=xC zDESQU1(fi78&X{S9d_hJ@jxiLw?? zOa0IuWurEeg&Pz}K_bG#&yT@QZ4H^=ueYegYn-f;S!_KO&_`U*TSO{sirl7w5O%T# zKm}H72&1AF#jp!*Nr)kUM8KzsL1OK1&w3`!d2E&tX^xw12j%w1*|BySMD@HEZWPqV z3&d2&piFO!O?I{{Kb^gYj<+^K?p2Mr&5HRE?qX#IfGY1WmUZkt>oc;nZ(&??EPLB{ zra(bzU}FtcmYwuO&_yx*RA$6lEM?r+=O5dUOzkax@_M|w#--4~QK|jck~Dc!?Ew=p zc$O1!r)^R^u;Q};D`bHM*2CUngltw8xXAMv;~4YcG5YmEDAU=(Omja%30mOn>5x)?)d(RB8lxTfYBNm_Wfkgf`f*f-E{>v^P@W(?1h+)6_vJ;)3xCzMeWZA6-G%Pk^pDmqqb_U)hir;|H+5=Z&t~+!6~(!ZooUtks&kzsK7~=)+Nnm?P12584^Y)z_XPk~T8?Terdm*}_V1 zjNKYJvyMB;3n)N!5<$6dNMQo{jvtt_?Uj8a(l2I^4|d|8{B7_IRb40L;0;G?%Em_& zwGDxYyepVXml_TnuUU7pJkAt1> zeAuY1O{nWL!+B0!k!(p5#K6PHaiQX67UE>Al(?CRD7az5Nf;!gAiOw?RjM(9$`>J$ z$v9%vtn$KU$;||d{<>*7WmA>dEjAG$Z}P-MtKW;&OKjg>my|~8(3C?}dra>N3C$YIKddhAdrrfvW`L|Lao9Clwq0Ib3~3Tjj)&*(=>^UHoaE~P zdB6Fj_14AZTksK~Mc4(x_&+JSRZ+vVTCyT9EGvschj=x=5WI$Kk%<4A>uT4|T0i8o zZ3X}$!|E|m+75el8rUr`5oO?_7!fk&1*|ynzsOvZx>#idF!U4VXSS0ey>cYfeaZue zgr>;~uYZ({y!{k9-L9`79c~Uexl6x^I^Gtk)M2W}&@JaI@PQhj)Xln2wXg%jJw{ru z>f`XUl}dS{*5~bZt4|H|v$0KVDW>ZyqM@667WjaM=ubJI#QfP))b%Ie@B96bp9XEH z_gF|0XIM6(%*NOSIlvrO%eXN-lU3&<5br@^B>T>vtzeACwMvjt#L zaenUo58-8t+15Kgp{pSTeTssHDPFk&S{a+aGZ6d~e#y_!Gvw;BkWAMxSxSUz2trz; zwA_&C88t9hb}2=@rh&uR3f3SfDNr(UEsS*Q%@!TtLh6mx@&*lEZqKRCz=~6etU(Ot z4QuaJ@O+izpfu`(w0cgFdM4RQPNK=HVkqn3SEFo+)H=%gqMPn7ub>&=3v!FCJ(;Dq z=fVeWTT&c~PGoj>`*KwbTqre=@fKzS82mdnk+lIk{c43+npN^PFY0WJg$TvfgV3jFq6TmOyy{0PE&x-ci zhl>L!q}f_W-BmDDo!mPHWfGt{u$rHq2Cjj!jD}k6q&PTF;Zh_hlH8ZHyfZ*76yx3i z9i$V`QElY?0R&9RTm6~NrF^W4@97yp@^H6cn<3mPLLg4`xclT?Lr5NLkkX6tN<4UN zF*~YV7~9<8_pea_;~du%u(fR!I)lz|FU!+~ot@Fe*jEm(t6X6dHCFOZ{5;qFo@O8c z$Dwi+-z@(Y!HwKUr@~10W-@~5g{s-z#N?sS#}d*2 zPU;olU?sLLvcoH?-eXD)Pcr8NO?*oFVp*~4H5fg(Q->z=O5`J~e-SVaC>9?_bc7Ek zz;chfwTd}`8Ghn#Gff04_h=AD*nlN2k}?Z?Ri(N&Wb`?DpKzX>z`*Pbz!(!wvjS&( z5n6BD@K|uY=WUW+QJyww&o%ThR^?Nt4U$#m4Aj}PG~6Z!L_iS;ny4G3`ftP&S^pQh zMKGP*kz2spjZ_XuomBP+$W`}^1Ba-Z>TFu?)jZ*kOD=3iteUXLoTU~Di@FBboyfc} z#P#b2@wcGI|Khe^TA=A;MIf}w8?~ZL#Lm3FNAq|>z9p|7FVc4R3xpsbmzXBvj}@3` z!=e@t87}Piht56BVia;md5&4c^H(ukU%&)UfT$lJ6L#Y_Qrsg5gN|&aOd!uf1W2@} zmJdj0?+v&?uFaRX`I#4+uw`~pzyv--!QR{1%1&<5ky3I^GC&ib8+F_xVx|f`TdWw1 zZXbpjSMbz6fUj^N9sZU+d&V3#9QcEu@S3v0?{tHNs@?OK%3#YVDhmWuRrgz}LR#p} za)>DMjXz|9lu`#z$%HUq{%9;7y5~c5rULKX{Qg=}US*K{ID$u3ocA@sqR<0LIJ!-o zWFQr3x1w@%IuOadOO{*dvASu%7U^dRkM86=y;hF0>R)bc%#Bc>&%nDlfoUu!5k8uT zBZ(qT%~Hsm@Z*p%I$~X!QFQAW=WZJfY_u?ejA?o-9E>SqOq0Xma5x+ehr{7;I2=(FK@t>E5j90pw3wo)v2ZY^1r$+GG(k}W zMGnT}0YwlLK@k)|5fnvH05aV)hXQdBP|xFl-%Vw~c@Q`VI0&Es9H@5wjE*mbXRBF! z4;;u?dSS;%IFA%X4u`|xa5x+ehr{7;L=TFZBue^H0B&oywfvr)hmrztEuN6hGf4q( z9!UzoyYoa+09@<3h@?o877Yf2!DuuVQG*dh6T?A8Qp8wD(PM%ph|y>~7LEr4Vmua( z1%uI8Ow-g@EU0K|EEWz4ni>oSqtRG67LWu<33Jsyw8K#%C*cr>EL!y!4MD1sJK!$DCD$3tQ~7>kICribK^9@K*oH5k!@dN3H2gL*I+ z(zIYO9FA!5U_7J+BrzD0gL*I?6N3>w3x|X8Ob+QGQ4fa0;fN#$gm57RfRX|*Ndb_g z036@|&jGMHu>o#t)pLLYZ1Ri-*fhpnJ(X)+HMg}z0H~N23@Vx+sqtV)3&w*XEf^05 z18Oi94aS3dK+|IZMUi8oq)3_`k7+?k4k>ah7LY?~P!8%5EhfeyYEaY!QIy1RFrr6; zay+C*6j9Rxf*K47Vl<*@T0ElZ@sOs4L{U*=nx@7SO^xaCSV#}ZAuSw?M`O`gIHpNr zIHG8Rs7RWo2eep75`&T?hXg^?B8nbVWATtANm@h?XfZjYC|Xbv<(L=?DUu{dL^Y(T z5lvBp^{CSPj%pInM*y-59FT$(Ld-GB84>_MAP@=UD2QPYh+!B8KoEq101y!X0)PU> zFa#BvvkryMCqc#jI?LXU2Nf2J$!e7cTpoS6`c?Lr;lw-lRX`IH`pM$$jBiQJ!W2S5-QB4M#g#jzV(h zBB2bu1GzmUR|7FZYcda;aiMp%C!y7*G#ME#y@sKUByeEKD2Sw^h#8eYym^sBKoGM-8+F zHzc*T1wPgeMQ`$~P}${GMjcxG1P-R$31*EJ>QND7mUfyABN){zW^Hu-FMQLe@`b}@ zvK_RZ6^nI|SHaF#1~H(<$&UFjCkRU4SnjB67m}Ko`38mLJ?SvobX?;(!O?XSE8r)+ z8Bx^nacnOT(>|O}Cam1qq=Y^IwT>56X_z{zF2SI_PjF^$uFMG*zDU}nbhd=;K<-@{ z>rc|o=zo()35Yy>y4`L5S5e7ARkq9gHsSzX0PWB6Or?<`z2qdIm9=eCb-_rFnu{+4 zYa=-DcUO`+ACAM2wxTrN9xEqpZ^A>YELJ-%-HR0r6$7EMK>Ex%+J@?Uj1kdxhM=Qf zZE5iVllf$X6nhR}1kE}Hyl@J!Y}c7$gkmuTA%Sjh$~j{C?>o`Kb{FaDv{o0XL%d9{ zGs>JIrrR`0lnc_z8x8;)+vHk|9$Mpd4}hpRG7~1MLFXcfJu|c`YsbkQFh%lY;XNXU zo`C+Ob}VtjHb@VfB!7DJFPtj$lUnrEcIq+#sZ%+jUjrM-0Wr&~ncx7Ym;E>Os9gt2 ze!l?sSD?kiHWgdbaL2$=Sw+mW3IzMqWHVI?7iP{zP5roF7{+KGAw*eW z*=FJ-ktJZ?LZ#~-OMzMw3uYPEk?t0bIJ}sA8x#+y3W4Plv+J71hPAG*F*J6&L^}x9 zfpI`05>bxFhavucB?VDVnV1vJqMM~r)oR&XCCXybs-|q8r-`V{ve{-*PW@+0Wl~zi z`N4~eFG5X}6Jij7VWOP4hnGP-2XGBoVP`LtFqGd*{VGhjp@g3J&eDcLvmoG_XnZ|Gz#+oVic#EYU4*K+vTy)7 z|J6-cOw|%LsY0U9q*M!CXHd>CN402)pWbf)?M2Mjtrv1@<1gRR`7MRDlj(pDB2%jx zXsChr5x7`EuuqwMrlfe`%-QIa9}|pWjK(9}M*(WM#0c|JetMx(R2i?WUJM4)>#dKv zOQ1}4E9v-LaP{dZ_^#kIcMwr87P(?;@xwQNTwn~ zPTWEx27H}SWC4SH70Hv5g$pNlM@juSF%051A0eV(hTA@805n%d0$e48g_`b}nt@Hu zSnb0H1|)CKVY|@fqv&49-u)_+QI{*naeb632gqqJoD(&5JCA6#6!X8BGKQL-XtybT zY~S{v(0+)jo_D!Imx`V1MSOa|29c@6u;1-<#K6}XMJzCwSCKraS-9}zc9hhQC&nOb z;}J$IP}=mp60BKOB+xIQ{c>I;WGu=nywq1bC5`;vg)8wL*7T=vI49m;01s1ZGesOr^Gd#3oWCmVKtJ(3~@~O#1{Ni zhqYlkw}5#G72Pt%?81U#JQUR}bL`(Kae%SW4sbX**f?XI8|Zf;7;NNCNG~Z$1S;5G z2p)LKlgP#S=bKq4Pk3s4m5sSICjr|U4J{liUS6Ft5X}oba!5z*c@qj(601k!U0AHC zadznnupstMT#K2}vGD)Oyh&IbE`m!@H~$U{VCTW19&t6gzM<@NZ*jOxoK>pCq3qnX zLz`j{14IV!BMdkXcZ<+yJt*9zEdsuG^BxjNrf$!1|HW0GB&qAq6Ap z@&1S=1@12Xj)a+&IR|UI z-XEDOy~#`r6XZc=o5!w>`Da!VX3+7 zq8@VHeGR~UED)^`{&Md3H->87%;s}l(0CY1`!|` z<&3Nn`|UxY{SZ|>?{bB%fdk_0X^u*2b(}ka8PvZKDa}Q?M%58!HJ94bdE9 z9tgp8kn046ogeUg$P)GTDp?sU`3ECfQa=fo3^gzLxuhu*YY_hjfcn-@7^^sIcukv{7A8&r literal 0 HcmV?d00001 diff --git a/tools/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala b/tools/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala index 92a1a6467ea..e34ed691ea5 100644 --- a/tools/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala +++ b/tools/src/test/scala/com/nvidia/spark/rapids/tool/qualification/QualificationSuite.scala @@ -28,6 +28,7 @@ import org.scalatest.{BeforeAndAfterEach, FunSuite} import org.apache.spark.internal.Logging import org.apache.spark.scheduler.{SparkListener, SparkListenerStageCompleted, SparkListenerTaskEnd} import org.apache.spark.sql.{DataFrame, SparkSession, TrampolineUtil} +import org.apache.spark.sql.functions.udf import org.apache.spark.sql.rapids.tool.{AppFilterImpl, ToolUtils} import org.apache.spark.sql.rapids.tool.qualification.QualificationSummaryInfo import org.apache.spark.sql.types._ @@ -316,6 +317,92 @@ class QualificationSuite extends FunSuite with BeforeAndAfterEach with Logging { runQualificationTest(logFiles, "nds_q86_fail_test_expectation.csv") } + // this event log has both decimal and non-decimal so comes out partial + // it has both reading decimal, multiplication and join on decimal + test("test decimal problematic") { + val logFiles = Array(s"$logDir/decimal_part_eventlog.zstd") + runQualificationTest(logFiles, "decimal_part_expectation.csv") + } + + private def createDecFile(spark: SparkSession, dir: String): Unit = { + import spark.implicits._ + val dfGen = Seq("1.32").toDF("value") + .selectExpr("CAST(value AS DECIMAL(4, 2)) AS value") + dfGen.write.parquet(dir) + } + + test("test decimal generate udf same") { + TrampolineUtil.withTempDir { outpath => + + TrampolineUtil.withTempDir { eventLogDir => + val tmpParquet = s"$outpath/decparquet" + createDecFile(sparkSession, tmpParquet) + + val eventLog = ToolTestUtils.generateEventLog(eventLogDir, "dot") { spark => + val plusOne = udf((x: Int) => x + 1) + import spark.implicits._ + spark.udf.register("plusOne", plusOne) + val df = spark.read.parquet(tmpParquet) + val df2 = df.withColumn("mult", $"value" * $"value") + val df4 = df2.withColumn("udfcol", plusOne($"value")) + df4 + } + + val allArgs = Array( + "--output-directory", + outpath.getAbsolutePath()) + val appArgs = new QualificationArgs(allArgs ++ Array(eventLog)) + val (exit, appSum) = QualificationMain.mainInternal(appArgs) + assert(exit == 0) + assert(appSum.size == 1) + val probApp = appSum.head + assert(probApp.potentialProblems.contains("UDF") && + probApp.potentialProblems.contains("DECIMAL")) + assert(probApp.sqlDataFrameDuration == probApp.sqlDurationForProblematic) + } + } + } + + test("test decimal generate udf different sql ops") { + TrampolineUtil.withTempDir { outpath => + + TrampolineUtil.withTempDir { eventLogDir => + val tmpParquet = s"$outpath/decparquet" + createDecFile(sparkSession, tmpParquet) + + val eventLog = ToolTestUtils.generateEventLog(eventLogDir, "dot") { spark => + val plusOne = udf((x: Int) => x + 1) + import spark.implicits._ + spark.udf.register("plusOne", plusOne) + val df = spark.read.parquet(tmpParquet) + val df2 = df.withColumn("mult", $"value" * $"value") + // first run sql op with decimal only + df2.collect() + // run a separate sql op using just udf + spark.sql("SELECT plusOne(5)").collect() + // Then run another sql op that doesn't use with decimal or udf + import spark.implicits._ + val t1 = Seq((1, 2), (3, 4)).toDF("a", "b") + t1.createOrReplaceTempView("t1") + spark.sql("SELECT a, MAX(b) FROM t1 GROUP BY a ORDER BY a") + } + + val allArgs = Array( + "--output-directory", + outpath.getAbsolutePath()) + val appArgs = new QualificationArgs(allArgs ++ Array(eventLog)) + val (exit, appSum) = QualificationMain.mainInternal(appArgs) + assert(exit == 0) + assert(appSum.size == 1) + val probApp = appSum.head + assert(probApp.potentialProblems.contains("UDF") && + probApp.potentialProblems.contains("DECIMAL")) + assert(probApp.sqlDurationForProblematic > 0) + assert(probApp.sqlDataFrameDuration > probApp.sqlDurationForProblematic) + } + } + } + test("test read datasource v1") { val profileLogDir = ToolTestUtils.getTestResourcePath("spark-events-profiling") val logFiles = Array(s"$profileLogDir/eventlog_dsv1.zstd")