From 644e4b4589de354aa591c90a3b1497b984a514d4 Mon Sep 17 00:00:00 2001 From: li chen Date: Fri, 19 Aug 2022 17:43:34 +0800 Subject: [PATCH 1/2] excel conversion --- .../src/main/resources/dbToMiddleLevel.xlsx | Bin 0 -> 17934 bytes .../sharpetl/modeling/cli/Command.scala | 51 ++++++++ .../modeling/excel/model/OdsTable.scala | 1 + .../formatConversion/model/DbType.scala | 21 ++++ .../formatConversion/model/OdsTable.scala | 28 +++++ .../parser/createSqlParser.scala | 113 ++++++++++++++++++ .../src/test/resources/copyOfOds.xlsx | Bin 0 -> 83429 bytes .../excel/parser/tableParserSpec.scala | 27 +++++ .../sharpetl/spark/cli/Command.scala | 5 +- 9 files changed, 244 insertions(+), 2 deletions(-) create mode 100644 data-modeling/src/main/resources/dbToMiddleLevel.xlsx create mode 100644 data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/model/DbType.scala create mode 100644 data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/model/OdsTable.scala create mode 100644 data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/parser/createSqlParser.scala create mode 100644 data-modeling/src/test/resources/copyOfOds.xlsx create mode 100644 data-modeling/src/test/scala/com/github/sharpdata/sharpetl/modeling/excel/parser/tableParserSpec.scala diff --git a/data-modeling/src/main/resources/dbToMiddleLevel.xlsx b/data-modeling/src/main/resources/dbToMiddleLevel.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..2d2ceb1c5f33d318e25cf71e882f6c739f591acf GIT binary patch literal 17934 zcmeHu1yEe;)+G+X-3cy18wu_f+yVr54ek)!CAeFH1()Cw+}(p~u;9U6rXe$PN&d|J z-`sjNRa5m&s*-b>K8uyzXRqFSefyJ>1cyKefq{Vm=_y230(o*^1J9k!84PT7olGr_ z?f%yZBfYbQ`G*lr%QgT!^a0u~qSB(N8aA2`y4Hvc0CAbz5-%#E4>4=rV^o6G)D`r( zML*x=k3mcO>I-2#G&j=ia#YB77?p>bq=-MPD&nZveWrALyG1o3J1ws(ab6Y=H{_D+ zxoYK>qqv8*t4S#Ha4s{!Z;n?YtP$lzbLKxA5_eP{D%=)1;)h9g?Wj$oSlMm`PX=E- zzLTEvt9}W4F7N^Wik;XGQ|}7$+-SKdAG$=INA3o+%hQKB1T2s>p1OAD#f+HXE%!6_ zJKWQS9)+g93GJQ*^lnfB*>>7Gp?5ujWfWRXF@~Z}``$Gs-~`_NJ+rXK6jVDYRRx-1r9xq>_BbNCU%!mRJpc(gqaYsQBP2EEB}%!58*W64 zp+@VR3>4pVEfNz>#`vLrfBreQKKsWD>G!c4nL6tCF)!|U@2WZAE;G(TO@~2Xt4`0= z2qy*7qj^>pr8~SaHc~!BroKIvZ-U+?!jUc{`3Ox0{~a_}_1lzRRf9jVV?C{I1PO<& z@+K~cs!Wxs_FjeWRpudF+aYa~VDN{JWIo#BIlg2%34TflbX*9BYw_XiafIFN9|WFD zVOgnu+!`)mc+X*Rn?*bMrt5=6R*`DT@n>1?dsuCqcj`+f%x_u?#&02w?qB%;K&<6n zhn%mRRh#IE->`<>ijK7lJ)-Bdwr$hbZ}VrCK<%st!oFk=QCVYME;LH+5HJdgLZ--)eZG`Nw?u{H1(*GwLm_)hBk> z@=#nYmxBfL{DQBka!C`f@hyE`tG<)wUWS+!)^Wy^pp8`||D+^{zp0W59!AR|NFD`V zj3!URxEClxXCIQz3>|_7K9b}=B-cCdz0v_|mrr^N_mt;ZzCHO*v~beh)35+f^#zHT%^rz*%yT-!*8RcvQt((Z?CZXE+N zArTuGDlL{xZV3Z(C0>{}#GJ@{KHji>Zvq!yDWw6=K+BoV)D2X^E(-5+Yd>U zw3n%ZrvUwXfsT7%#{dm;jY!$~0}KSjAO7z;2-2U;*psBuwly?=nmY6k*I~nwux$Wj zsAV+gK!9t^q<;|0o~WN?_H!Mv93=DSWQU`3DPft*ZR-QWZtdDHk(ev+{Mk8ZGS(mg z<$YVdh^RvgHjUtzXlh{<6=b>@Fw%_$GA#OiY&NNG1~_H%ZHduKa3~XEP^3;D&1K$U z!RK%X2skTZ;ftAv=+U(0f}SSys?iZweu0>z`*>HTxg;BcD5HrmbF%6`0{Wv)g@O@2 zmdyHYwy{0h7*kj?_;bi9+CvPA^LC^wfd4~kU?=t8c2Q4a@F*NVyQqYP2jTGlDI8v5 z;zSY(7V1iK^>3^ArqF@PVQmnT)^v+2w71MCU&)4GGjwURJ-}$-;8{ZRL#&QUwc>1) z_Jn74t%xe+yz2gX>R5BdHFGeTq)W&iQ?sj8tyW=8oT7LE&>PS#kv)SA&6O3?E zmIH;6xh&pgZ~P&r*}D4O^B|sF)eI>IjAcM7w1YCMhpftxI<|Q|o!wQ(k&tvsb)dDH zVX?XPZeQa?=7h0VVSg%fYxN$j@hN{a!5S#fcg2^np|}Od>YN44cQ(^Y;%i!thsw%^ z*ZEgtiL0}G2Syx#eS5qQ#xHglA{6mvorNi3l7#MGx87% z6EgaeM$6D8?CZ_N26WQ#>@!vOZ{Of_*^^b&R(!A%G{^0Mbv#@UtlMdN);mEL=6O_Ci|Latw_=_q z>p_s?O2m>knRn!%@{>=-pM-EsN2??R3*g|Z8sej_?0b}y*=+{X6$vpp>0{y6d!^_( z)q80#3)cj9Q6QT+rSR_x*W`EQAlH|cZVjrSCztvmr%u~=naa8rnF^jW~ z3SA_d5ou3^_ENZ0fzcaS9Xkvrnv(?SKrv2-X;swfE=GzI_#}}mx7e9bgp(Nf1nLxR zusVCJ&tuz3A0dZ6FB>LJFak3#57fB``e6ESIu@vN15?(DD7{OeAH%oX6VGEqKiH8J zA$V^7dhb>2<-})zaJ`ytGv6SM&lpMyTE7~! z`}HhQzV#NHEd8AU81iZRhk{obcR7on(~)NfoL}8;_6dz&1dv}0i{@lg`8V13y&5o& zM!sl@$LD2sfe;`}K1u2&!M*|p<5L2S#Y$(6k^zV_Rz#W zxF%gK#iTLOLsvNZxb6)21vF=5A+&SP?k)JMm{haKFGRn$%yy3QRquU~9~~{2lcKnD zCa&y$MIA48!Xxr5wb?1HbU7GOK2A@m>>cPs5d1+;F)IJ5DE;&!5@#&uIMGF#)9;Bk zdE${?I(zM{OcL8%E2CTdsGuG%Ll$QtbLm-1z!mKMo)~7Mt>@x;Z0$XcS#jky$o!qQ zpIWJN*1z|wgF#fDrdmVMLKH)^t{Ra zjFJe+;lgO&fz`K};pHREyGlvjRd;Gl&J$&^hb0!V#?Mn^NEsr=+xDnw`(M4yyXBQ$ ztB{bXqunkMdr?t4c`-rEqx@syh~`B?u6RCm)o04!zH5nOqQkY9QFJ3WP*k$diThTM zPVL>s$RmX%jn#roeBssn3sS6W@5o z>1>0hD+ow+uhL$L^WZyq0_% zq}4xtuMKA_RM1QEQ%_(p%?O5`kEZ+!tj%*vj0kxxq?KYWHzL+MO(YET{l zf@n>ceOXCOyk70;_d5oRmbR1;8DUn|38akAB$HM~(9Ll?zAsUI1QW_afD%1R9!@}A zhlSRfHpm|mA*Vf|&!iIEU6kC;pl0m$AQs)58a)L&Ouv3Q6$1kq*CL93~}i6xh1 z5|?52c|0k}Y<|8dtSgjR%4O=kx*ty2LCH!Q{P1$Cd2m~Rur3R|+l|(&tED4suz{$c zKUFG~kLqW>r=AHLDB#6>~$*#gPA%>RfWoy@giioKvU*`6xh?xc>vHw^Rf7*)iPs=oHw4OAW z3CF^>u5XmC?e=`eRXW+rrK5pmTJ&RsH^gqtE5mJP5Pi+e&_Eg&5^TG(s=Dl2IUqdL zcU}xE(#9A1&SNPIWnksUitokaJkvZRWduF)c(P}iy%a2N8z%!11gYm<%Y2o>H!<~? zdDa`u8gWFB@B4|&D#Y2ljYx$^MrBJI2G0dfXxEW8D}tnuo$eHp#8jZN?DN@fdXvcw zymhLf0#(^~h13B}FMAI%k8veq`CmzDujCJT0+UBLRx+6&+)hvIE)Q%@JIirS2z&t*#x#bO%C zz$7i44W~SYwUr-|(thuj?$2)F&Rjbc_U5c<>J&->&0r_FlTcK)pyF(s(F1KI%ford zRYY^|^|2zI>;An;VpAV73t>NOuZL(^rVGGd zPnQyVQfDy$fS2Ee(&J}##IzUzgv#%-TB?mx)~L1v#LMuii3hRk!terg+bkDzUb@?+ zu)`mt*UcnOp7dI59-!Wh`%4Ct;Yl)E`UXPVLK-2y745aSI>3zJ^pxKir!bmqAAzxCVH<_+oHm5ysMZGCu zTgVEV*&FifI~>c<8G7U~`*w&?{W`-c_ljr*~#^}_Hx&aW55!JwLmKso76a~Satdsi%Z@;7>T`Zn9XDZe~@9G(E7@g z44*iZM^R9RVZTNsk2Lriu>U?)^ivPlS|PRVOk&8?t`g)IlNnraKMzj|aCQAq@Tf#$SQiux-0^S$)uFoza&~_|gz6QSS@1{n z2s!T$V5%Q|yAW}m$hTC*IZyNCn8Zu;DG|B6r;`Hxqn$xg*v-Mp?2zEd5Bbp)0;L6{ zH)=2?RxUVM?WW=N%7AalSd<}fgf07dI)v{CRfYz9w@~8;u3db+pQoMjX#p@kulbPV zd~?CqDpGaDhTNFGl7SbuxK9H@6*hABLMX(cd)yZRQZZ>YDbbRagzWIEv7iU?F{wy4 zto-}TfuH?`hL11?mkmzJ+5w7;2X3$J#P?$YbxRaH6RphXd)+Spp_m{`(fDv#HZQ!swIEN}9sH3NYez4YuKXY}JE{l|=!`~M`Pk68D}{&7bC zT!4A>XMr7!&K>CCVx+z$v4n#3Yo+m;w=4<9urj`|?B%`Lo_rgqT&JzlEDX=suXi2b zomXl1WtFz@sjwgm6X>OKN`!Vo*F|ot?LF_`eMzYcHyPuUl~f+jz83IzZN92zXS$=W zj%5vJ$?F_jU>&sFcOjV@qsBn1&Z@dGYr7ZW#0p@{Ki+Z(uQtG>(}hIy2039Xw5Xtnay zu{flCz(Ixc9P6G}jhb^o3D?Ezk*r#F$Jy?SxU%!JHI92l`+WCp%US~?*U{2#=slEW zPL737*X61eM^;g6>KvjJQJ=d!%d=1eW7y62n3Y5J?X0=5zQ`zjy>)uG-4t=lK=zid zk>Tx4M%UEKR{rC68f;Z374M{q_SliTnsp%Y8Z7;{I1L>w* zP z)KuU~X;I5;q5RSZ7cGOWgpNKa91e^WU}qW@j6CoqqIOygd3Tj`Ita$U9@C|LWYvYL zi(nI)$j0~BwU-eYjS~6amq{AY#eU*R2k{*xenfuiP1A=KuUUV(= zhiUQsjBRVsSi%ZVq^p4VXo?EZ7?uJP?VBZpp|OGUwW=|cA#_;;s1<^en25xNGdQhl zeKg&Lu`U{bOmHhS;xML+mWhNuNgU@LS)oSmw-WCNx-dC&!mXQnvDjCjnC4FJ%VB;a z`n3>kb*N*Lo5uZq!duDs5JjYB9V(^{KK{*C1l1iMDlE7kt)lR<#{Bd5YXKdW?TM7O zY9upXUNr@o58^4sy>v(UXrB1-W*A!S&~bIg-xb%HZpoWqv3A z5brzTLZ_XL;g_$xr$`!0%vPUPb}0!gz1=>itB)yeLw~-D_n+Njd`A{yVAiTGYp*c4BhzFSvs-)>pb+uA z+?oKNb4UCT1fz!vY>-*bJiWH20}GS8CJ#}3t!oIu5s{~-jgu@G4-dy2f~U;iL5!lb zthH+-MolFUqP{G+m13l#v{YN7q%^-Zrv+_hS^#yyOA~t+OmFXWJ(6}|I=hwLn^K0B zHuh5n-ut#P1Y-Lc$$^X{wJ)2Q#xO%45$+^x2MG!T0Du|hZ;?{#1oZ(rF$&1oZ*yAo z=j0&920$NAmype0L$n|cLO7e_OYkRuT^!O#4z^7%vxli3qF+deHrt=(gy4#sRZkXT zeu<&~BQi@#r%qrH0z0C!y;RW-Z+9d;SYoxz8N2n%tU7YQpb(lXa+7m^tCI@;jKOr{ z^vAH}m0Jy%v26xZ*&>Ns8{&^}Uz@uJyY0@99>n2#&30}KSkZXF{bO->%#J^bgMY}= zb)%Tbj#pMztRawC^u`_=cz2oaWJv2{kbGaF+I|Ej%_@2>Mfq(kb<$ObICGR;O$gNp zRxYk^MT)e`9_^|XG9DR??1b@)FS5WXi|+Ddl(@Lrtsi4p6B^MrfF7;IoWO%eEBwTxCHU#lT6d$tWzu$J>Rp081mX5-^7bE#!_US0#0A?- z3G@eX@Lu87v*K@6$PNFLW3-=>V>DDpF)d2XukOX?at%sWIrE;r*bXhONK1RXT6>nZ zS?g$kv$tGAsjR%y;H`SO*jH9Mtv6?BOG_WY<_flJr@NA<*ZMpX;%*&o4PBJi)vFT9 ztCM{))n=~0@Y3SvN}Mr86b5tlWL^%+#JOm&G4#L5<&0%obo4zM$k?*qShG=^o(d4T z6bqc)8x!U&TubcFY>JO%dcR|IHNnfHNN8~hxqZa)LMhwvluX`b>FWTI%^G>sf}f8p zT3^g7quB2Cxx<66iqDf(8$PM3H9H;SzLlZ5R>6GVv}&3t%4IZqfR)RA4WsRMKHZ?p zC`5U~a{d0`UOOg{Q?Qhpp{vT-P0fahWj$g7{)bqz$Yg)Xk#U_?f&PbV{~J}c2}Uc- zdv{W12d$aGu04hu3GH(S*H%@A54sxXYfFkB3sb%;*N17xzBALlnfAc7Pr03J{D#2H zpBD>Ab?~8rwwOW!4QU z-zAh`VfFF~&z4ePM0%IAGG)w8qkG3Ne@J;I+yZuv@}7lGX=(zwaAVt)N0T$x(IjnJ zN6<}1c&fvpWY7P53Y?4(ekqGizN9l-c1Nib8j1}qPU*gvL&jG_wu(B!U=2h^1RX07 z*OBc@csD*&7yMnEmOGHMvI+g=D45l&)E&Iq3U96*v@iD0ndn8?ox19n!JT-mzsuns zAcUa|eX!e~Z$W^v^m^apbRy-UahLAQca{R z5(-^J{xc7KI&bTyc~Ts&tU9S})LBg*~uiHrdyse@(vBOgs0({6erzx#jn>!P*TRPc1w4UQOH^jbo5!wmRL&o zh+fB*K8_5f96^%eD>Vl{+||JpXeJczogQbYdXC(TZj+%cQXKZeZ@PnITw~>==t&?~ zw^G77Tud|Jm3uJK8?Ajy!@>~Nq%nJ9bKxTfccJ5lo2+TJoYUb_)ua|&f^+gMTj$Iu z-n%^y_h&`D#-HahibO0SjoA^a4bR>_=;3F-!3qUzTY)szH97poWIRFV<_Wgx9;`y^O+k7 zJ-(K6W(Jk}V0Rqnrr!0v(z&Wn!V|G|P6+@qMP3ZY`c%DqWBl0pNDX~5{Cn8H7bDFTcBqnH}sh62SD3n-?psP(jS zaR0p0VmWMo{@;r!FhUPw;r1qm7KT6new^GHd^2Y`PlSBya|gcC*gDntqF%l{MTt;N zX}fxxx$H2zbsCJ-^sWip)xu3O(}>hi8*G(1keRt9vcIqy2NKEeJ&UoKiM9$dGUl3Y z-^$CIQ@eW?Q18e@|HO^ZqHky@2XBwJP(K?8GJ(Z%q0ux11jly55~ENtV`8}oBwk^z zbGIu+A*$zQH}dNZ@k0yAj;4>UL`DXrz6cBS zLw>QmL3R`$Jth*Yo*<_GRyzwsi6oJR|V+Y+fGk>E6_BDQgynvRhPd_vcF zA+LFdG(Rg`K?3P0IQp}oBr1n=dn40!u;bfk4YIc?3{$P0_&bctbvGaS)*_b$>GXFn z^yFW6gQ3^S1wNCp-T8!HVddgXy|Xp{0-9F6ex1!RPEb|V`1mZ6_2eBU+-tAT;@S*t zDn&GH1SpJ7+LUe8L)m&9nR>c>xnMX-^1ueG-0VUt8zu`Ge_l6A5pbIy!Y4`o%_}EqcE<>Z%B*T;X#T2p zC4REIAAcJwNf_nXnqT&Z0xOK(%_%mYesK@QOeFCOnElPD`xa{XyLoyv>>NSX8Ti+h z=KHMg8+QtLLr598Z@n@^e0IY1v^Fp_BJemw@zqTEs9ECM>YY8qORmdb{}@X(Av*3= z-@Q$4$mk<@vmUYN(C>fhmf-9v1o%YPm0ho1p=z5z7}AhZJ^071oONsWpq| zY~m{gn6cIgInV?oc=~Y%-*PQbr__^x41*Ud+cY`m^X z%e9V4?8j^JK3JtQoI8X4P;E*&b&^ zPk(C?tp*oy9S`gCv7lK@p7Q%Ni=1PQ@EdY~wD-EGV;+LpM*|*6Xw(IJQKAfOEngn- zrShOs_k$i<2R=L(a^J7darrZ$g))upB zQWW0m%G2Ry`>~&J8Lafa1MfI zUgvv-S9jFD6NDoa>r9?{Q`*p_9c#jXsU^Qc8KO@|EnAfMG5O+p*h!l~dFrHQ<&=@P zV&ld402S<_s7P{DBefCIQ^OGUm$k5OIr1)H;%N4rw}m@i+%cjx7Uqt?Fr3P?@g=W3 z_77ftpPPk%{R7h9I-(1If=JM|UOp}~iit=Xg`N~NVM>-rxsoXCjAy!u zz=OUnW{#I5h$wVq8uig+8vO+md2C*s5@b6OriLLADqKnB8%{GxY=h6sIPw-pRk;m- zAP5Z9o=ba(Zv7=p{f!`9wgWX(;jms3=A`WKs zZv_((eLny`z$P-QZ_~crcJIL8xGeRlH||%D{oITt=w6QjCteD1e$k0TJ<(8K+C-8j}DO$RdbSqdCgf-SJ?BZ&nJxce9Mc()=v9nG{N zg~l?CB`UXqQY6vC0BIaz3vH3mpCCkeDev> z?YKul$Xs-G|>+igG6+ZX72Y05+VH@=ho42MS^7^3J2-^gt{1S+pgq@&`f-NrBD9JG5 zT2%`$Ojb}u-_vE*p)^@0oJjHeLD7}1zNLrJnJbMR^aw0)^w3N$D-zzSU7lOcD$^$q z(`*y6I9-IMp1<~2$aIR#zQsb?>mC!c`T(1Ndc?MM(WMr|bdMgf+<1u)EC|psg<50&Guil~aV1)a*`asZX8^Z_`e8;~nFM%%KL#{o{H|kjd*ESyZ*~JWorvT7J z)6#!X+II!s!YcLMtX>& z4>z=>&a7oDrO$;FjPBN3e^nq>1so#l@^x}aElOWGlF)uGyN1$>XP;$0!cQL%AEt|> zpQscW{xS?o;?^=$VhXVuLxZ|BNGXfMxLD zRSCung2VO3rRCi9;k(94W;sT6|F@sL$(6CyI2qOqCw{olL`)@9<|E7N1&vxWV* zqV&nNL&5n-(r!DT6N!&3$NbRL!V?CmOJH*C7Y$x{l3{&4Vw) zfMf=6a&-mRK>VMlS8;!Sx7hS!kEWtes8g5YXBY_raP31-u%v7}a60_yn!EFPSaP&>NSt;Y%A{ zF8Z;{rfZM|{OG+@H3MX#dlCp&DEMM@abW@7fjHI=9_lYlJ~4Ce>E)7f#apag-cm%t zfqN`O4w;s+IPSv_2fg0IYff}0 zV1VPH{w}~k9}MJYbpK};CV=PzK!zLvjzSxqDOl@%}YTw8owH+!%++)?_?~l&?Qg)T%b-Z7kF!{qXsGGoGA#V$mp=TNa$o-u3}u z$`?yHXA+0B(Jn#^?O-xVD*&tt#T^;nVo(=tMoVF|C$~!D<23jus|=!Y;D3bbZzrK2 zdN6e%D@%JrOM7ia7i&X1%?Gs@j9-vxCqnm1YjHBEeMvL-wJy=2{@}ZU^jBzpcFh!d zBK))SvoS|zoBUuh+OwS4=j=;UtoVs^j&V|~I%Qv}Nwe(5P~|kNZQ z#(ofu<hKJCz-M~f)@=q&Yag>f&qd+(CFR0p#orJ{*nKcPPYl#$2J zP(7{Uy$j5@LyEVJFw3I1_n`B6I{+^{hXqLMtbIw~)hQqFiU- z>nOC4e?)m)f&bT0oPcui2QH6GV*ffy8}uJh9=-E_9R&ecW&eT8qiOUz%F`VozdkQ2 zza)5X;I++4*2*X{xRpC4wC#z>-hQa zC{JkpH9_+`t*0y9ztXxO{2lO-)*qAj=~D5pwDN$LzsCFtt-sdDf2Z|yjqO)j#bmz& zKGOQ5Dm-1J`ju8E&jDH7sYDqk}Uw<85jE?~S#S8l#;c27wU|sxmK(hQq_}@mx z@3%g!;vYJ(zYZI={~eD1)tCK#^V16Tq0s;9kmY#1`DXzCOWFTB&eQVmp`QKgDCPX6 ny8Zper}f9Lu}|UpW34DB2?_jq1OXug{>g>`0eQ{yaP>a`xuv!* literal 0 HcmV?d00001 diff --git a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/cli/Command.scala b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/cli/Command.scala index 91ba0cf..5535091 100644 --- a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/cli/Command.scala +++ b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/cli/Command.scala @@ -4,6 +4,7 @@ import com.github.sharpdata.sharpetl.modeling.excel.parser.DwdTableParser import com.github.sharpdata.sharpetl.core.cli.{BatchJobCommand, CommonCommand} import com.github.sharpdata.sharpetl.core.util.IOUtil.getFullPath import com.github.sharpdata.sharpetl.core.util.{ETLLogger, IOUtil} +import com.github.sharpdata.sharpetl.modeling.formatConversion.createSqlParser.createTableList import com.github.sharpdata.sharpetl.modeling.sql.gen.DwdWorkflowGen.genWorkflow import picocli.CommandLine @@ -93,3 +94,53 @@ class GenerateDwdStepCommand extends BatchJobCommand { }) } } + +@CommandLine.Command(name = "generate-ods-sql—automate-generate") +class GenerateSqlAutomateGenerateFiles extends CommonCommand { + @CommandLine.Option( + names = Array("-f", "--file"), + description = Array("Excel file path"), + required = true + ) + var filePath: String = _ + + @CommandLine.Option( + names = Array("-h", "--help"), + usageHelp = true, + description = Array("Sample parameters: -f=/path/to/config.xlsx") + ) + var helpRequested = false + + @CommandLine.Option( + names = Array("--output"), + required = true, + description = Array("Write to sql file path") + ) + var output: String = _ + + override def formatCommand(): Unit = { + commandStr.append(s"--file=$filePath \t") + commandStr.append(s"--output=$output \t") + commandStr.append(s"--help=$helpRequested \t") + super.formatCommand() + } + + override def run(): Unit = { + loggingJobParameters() + val createSql = createTableList(filePath) + createSql.foreach(it => { + val workflowName = s"create_${it._1._2}" + writeFile(workflowName, it._2) + }) + } + + + def writeFile(filename: String, sqlContent: String): Unit = { + val path = getFullPath(output) + val file = new File(s"$path/$filename.sql") + ETLLogger.info(s"Write sql file to $file") + val sqlWriter = new BufferedWriter(new FileWriter(file)) + sqlWriter.write(sqlContent) + sqlWriter.close() + } +} diff --git a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/excel/model/OdsTable.scala b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/excel/model/OdsTable.scala index 397d6b9..5712dcf 100644 --- a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/excel/model/OdsTable.scala +++ b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/excel/model/OdsTable.scala @@ -28,6 +28,7 @@ object OdsModelingSheetHeader { val COLUMN_TYPE = "column_type" val INCREMENTAL_COLUMN = "incremental_column" val PRIMARY_COLUMN = "is_PK" + val IS_NULLABLE = "is_nullable" val TARGET_COLUMN = "target_column" diff --git a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/model/DbType.scala b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/model/DbType.scala new file mode 100644 index 0000000..5034690 --- /dev/null +++ b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/model/DbType.scala @@ -0,0 +1,21 @@ +package com.github.sharpdata.sharpetl.modeling.formatConversion.model + +object DbType { + val ORACLE = "oracle" + val SQL_SERVER = "sql_server" + val MSSQL = "mssql" + val HIVE = "hive" + val CLICKHOUSE = "clickhouse" + val POSTGRES = "postgres" + val REDSHIFT = "redshift" + val MYSQL = "mysql" +} + +object OdsTablePartial { + final case class ModelingColumn(source: String, target: String) +} + +object TableConfigSheetHeader { + val SOURCE = "source" + val TARGET = "target" +} diff --git a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/model/OdsTable.scala b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/model/OdsTable.scala new file mode 100644 index 0000000..e283b64 --- /dev/null +++ b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/model/OdsTable.scala @@ -0,0 +1,28 @@ +package com.github.sharpdata.sharpetl.modeling.formatConversion.model + +object OdsTable { + + final case class OdsModelingColumnParietal(sourceTable: String, + targetTable: String, + sourceColumn: String, + targetColumn: String, + sourceType: String, + extraColumnExpression: String, + incrementalColumn: Boolean, + primaryKeyColumn: Boolean, + isNullAbel:Boolean + ) + + final case class OdsTypeModelingColumn(sourceTable: String, + targetTable: String, + sourceColumn: String, + targetColumn: String, + sourceType: String, + targetType: String, + extraColumnExpression: String, + incrementalColumn: Boolean, + primaryKeyColumn: Boolean, + isNullAbel:Boolean + ) + +} diff --git a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/parser/createSqlParser.scala b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/parser/createSqlParser.scala new file mode 100644 index 0000000..1f5986c --- /dev/null +++ b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/parser/createSqlParser.scala @@ -0,0 +1,113 @@ +package com.github.sharpdata.sharpetl.modeling.formatConversion + +import com.github.sharpdata.sharpetl.core.util.ExcelUtil.{getBoolCell, getStringCellOrNull, readHeaders, readSheet} +import com.github.sharpdata.sharpetl.modeling.excel.model.OdsModelingSheetHeader.{COLUMN_TYPE, EXTRA_COLUMN_EXPRESSION, INCREMENTAL_COLUMN, IS_NULLABLE, ODS_MODELING_SHEET_NAME, PRIMARY_COLUMN, SOURCE_COLUMN, TARGET_COLUMN} +import com.github.sharpdata.sharpetl.modeling.excel.model.OdsTableConfigSheetHeader +import com.github.sharpdata.sharpetl.modeling.excel.model.OdsTableConfigSheetHeader.{ODS_TABLE_CONFIG_SHEET_NAME, SOURCE_TABLE, TARGET_TABLE, TARGET_TYPE} +import com.github.sharpdata.sharpetl.modeling.formatConversion.model.OdsTable.{OdsModelingColumnParietal, OdsTypeModelingColumn} +import com.github.sharpdata.sharpetl.modeling.formatConversion.model.{OdsTable, TableConfigSheetHeader} +import org.apache.poi.ss.usermodel.Row + +object createSqlParser{ + val typeConversionFilePath: String = this + .getClass + .getClassLoader + .getResource("dbToMiddleLevel.xlsx") + .getPath + + val typeTestPath:String="~/Desktop/dbToMiddleLevel.xlsx" + + def readDbFieldType(filePath:String): (Map[String, String], Map[String, String]) ={ + val tableConfigSheet = readSheet(filePath, ODS_TABLE_CONFIG_SHEET_NAME) + implicit val headersOds: Map[String, Int] = readHeaders(tableConfigSheet.head) + val (sourceDbType,targetDbType) = tableConfigSheet + .tail + .map(rowExtractDdType(headersOds)).head + (readTableConfig(sourceDbType) ,readTableConfig(targetDbType)) + } + + def typeConversion(filePath:String): List[((String,String),Seq[OdsTypeModelingColumn])]= { + val sourceFieldType = getSourceFieldType(filePath) + val dBFieldTypeTuple = readDbFieldType(filePath) + val targetTypeList = sourceFieldType.map(it => { + val sourceTypeList = it.sourceType.split('(') + val middleType: String = dBFieldTypeTuple._1.get(sourceTypeList(0)).head + val targetTypeList = dBFieldTypeTuple._2.filter(it => it._2 == middleType).keys.toList + val targetType = if(targetTypeList.contains(sourceTypeList(0))) sourceTypeList(0) else targetTypeList.head + OdsTypeModelingColumn(it.sourceTable,it.targetTable,it.sourceColumn,it.targetColumn, + it.sourceType,it.sourceType.replace(sourceTypeList(0),targetType),it.extraColumnExpression,it.incrementalColumn,it.primaryKeyColumn,it.isNullAbel)} + ).groupBy(it=>(it.sourceTable,it.targetTable)).toList + targetTypeList + } + + def createTableList(filePath: String): List[((String,String),String)] = { + val odsTypeModelingColumnList = typeConversion(filePath) + odsTypeModelingColumnList.map(it => { + (it._1,createTable(it._1._2, it._2))} + ) + } + + def createTable(targetTable: String, columns: Seq[OdsTable.OdsTypeModelingColumn]): String = { + val primaryKeyField = columns.filter(it => it.primaryKeyColumn) + val primaryKeyString= primaryKeyField.map(_.targetColumn).mkString(",") + createTableAboutPrimaryKey(primaryKeyField.size,primaryKeyString,targetTable ,columns) + } + + + def createTableAboutPrimaryKey(primaryKeyNumber:Int, primaryKeyString:String, targetTable: String, columns: Seq[OdsTable.OdsTypeModelingColumn]): String = { + var sql = s"""create table $targetTable\n(\n""" + sql = sql + " " ++ buildColumnString(primaryKeyNumber,columns) + if(primaryKeyNumber > 1) sql = "primary key(" + primaryKeyString + ")" + sql = sql ++ "\n);" + sql + } + + private def buildColumnString(number:Int,columns: Seq[OdsTable.OdsTypeModelingColumn]): String = { + columns + .map(col => {s"""${col.targetColumn} ${col.targetType}""" ++ getExtraContention(number,col)}).mkString(",\n ") + } + + private def getExtraContention(number: Int,column: OdsTable.OdsTypeModelingColumn):String={ + var extractAddition="" + if(column.primaryKeyColumn && number == 1) extractAddition =extractAddition + " primary key" + if(column.isNullAbel) extractAddition =extractAddition + " not null" + if(column.incrementalColumn) extractAddition = extractAddition + " auto_increment" + extractAddition + } + + def getSourceFieldType(filePath: String): Seq[OdsModelingColumnParietal] = { + val modelingSheet = readSheet(filePath, ODS_MODELING_SHEET_NAME) + implicit val headers: Map[String, Int] = readHeaders(modelingSheet.head) + modelingSheet + .tail + .map(row => OdsModelingColumnParietal(sourceTable = getStringCellOrNull(SOURCE_TABLE, row), + targetTable = getStringCellOrNull(TARGET_TABLE, row), + sourceColumn = getStringCellOrNull(SOURCE_COLUMN, row), + targetColumn = getStringCellOrNull(TARGET_COLUMN, row), + sourceType = getStringCellOrNull(COLUMN_TYPE,row), + extraColumnExpression = getStringCellOrNull(EXTRA_COLUMN_EXPRESSION, row), + incrementalColumn = getBoolCell(INCREMENTAL_COLUMN, row), + primaryKeyColumn = getBoolCell(PRIMARY_COLUMN, row), + isNullAbel = getBoolCell(IS_NULLABLE,row))) + } + + def readTableConfig(dbName: String):Map[String,String]= { + val ModelingSheet = readSheet(typeTestPath, dbName) + implicit val headers: Map[String, Int] = readHeaders(ModelingSheet.head) + ModelingSheet + .tail + .map(rowToColumn).toMap + } + + private def rowToColumn(implicit headers: Map[String, Int]):Row=>(String,String) = { + row =>( + getStringCellOrNull(TableConfigSheetHeader.SOURCE, row), + getStringCellOrNull(TableConfigSheetHeader.TARGET, row) + ) + } + private def rowExtractDdType(implicit headers: Map[String, Int]): Row => (String,String) = { + row =>(getStringCellOrNull(OdsTableConfigSheetHeader.SOURCE_TYPE, row), + getStringCellOrNull(OdsTableConfigSheetHeader.TARGET_TYPE, row) + ) + } +} diff --git a/data-modeling/src/test/resources/copyOfOds.xlsx b/data-modeling/src/test/resources/copyOfOds.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..770e9f925e7d2de52e3fa229707c1cbad7111cfd GIT binary patch literal 83429 zcmeIb2|SeT_djmmcS6e6Vo9Nel4S~M6G=r$hEkEGRCY#osz;I1BBeyxCWYpk%WHsbW+>zX6xbLa`4#14@*@&ot@%eS-TXe^Ko{Ib_(ohaM&#=Y9O}v zm5I7Q(?XXy=c395Zr1s~-lXZ^!z|-mffy#ea5356y!Jr3GLu2#)q*>*rN zhgsXi?PZ?%6Xf;_E8cEc>`yuE{#N&1N?pLv@Q8&eSPzi=R)%9K4v)0zw@PsLN+;RgOjL#lM9Mq7C{%@1sij);mr}X?Ke4c$e3|QxBm*ICA-Fgs670-pzv%3*Xf(%X`?SXV(5SNciZ$ zink-}%E~VygcbT%FMS$prl=67-tk0CbZ2~=ZBd5kG2{E~Pgq2WtM5HAOg8a-!Syln zb9Dr#@n>X86bqa2z{Y@pdl_t8aUHwSy6X5dOw2KU;&Sf}?U1>$#U9tIWZ!}_ClHZm6(<O^EAEHST78?&zrEEAH$*{iNWE@R^| z4&{!q)l@VH!0nGypFO^Y!ss7YDikk@vDHvyov&btw;cQO(!gl>&+PsOx%P~GnH~NG z__r;;|FR|YWz`tl7;9#XVeG;Ld)fgN**y#oH7f?N+%$a*E6wYKab z8e*DTJwuClojT`;?Y+@0&4ak!Ey}~tZ$>Cl1w$x4w@#m3W>n6mY<}zc*=6g?S(Fcj zkZsQO2Ap4iOGn;5x+FE+a-8>Y^Cn~M07`&`SX`N|V-83y*;lH4yf z!Mo|I%c%UYWo=vHxU&qhs#e0kvaIvRoYW+9VB==*G}$Pq!hb$fxtXU(U5KMe)H6)o zC#E_idg=C|Huk_<*o;;47IT3;5uAnZhXhvGxh zn&Qt5C%N|T$XOO$y3@On-28%%Wv#HniWN*w{ezra)`eg)8*%rwwG#D0)~~Z;Py*7V zmFUe3YR8pK6gf%_%VEez{F+!}?YeBm#kHAf{-2TElbg9!=#BgGOZ zTJM!g)DX=BMd>9Z(A7}GT@+V6Mo%(hfWn~nhhhB~t!*fJ>kZcedVl@F1YQ4;7i~6v zl(I}xcS0k!RkwgrVN)>jp`sh3MyJ2H*hnAhxZ=xbtw~@ERh2TDGi_+Zunvr0pPCQG z75Y62$$zLz4MC@$A{7jGn;`rG)%0k+Ha5P)6`bCBzSms~JS!AOKJ`I&O+rDB*wE{W z@QY-Ow0C6sQyXpcYD#m5OJxzm6mlcIEsQ)A6(#$z?I67`*@l5&^t`C4kzjOm(VY_r zbh`rI5x1d!#v`902HC_T!Oz>n!#kByNy{uDAgbGJe0`H$8wNFH6?^5+A!yAPbDgk^ zD%k=GCCUahK)gsKV=2v~MrywV17(Ayj$lxbWD|;LOf(Aytwt6VYRc#jGezi;E3glJ z2U-y*JguXVcC`oPN3Lg3so4D}J)Qd~XKDowHKKtXq6i}V$y-KRGqF$V6|t1Qi>P5L z$%Y~6OCF%UOF&UOOb5HjQ6z*nqaJ}M7-}Y)3L>yHiAFzKu^NLE=7G^0DuX1}POxFL zwJ}CoQS`oIMr$jXlwsPtoTR7I*ERV3BMCuyvAzN2*+L_p^I&{BM-hywBcWmw{PB(6 zpPGx83(6*V(LE5)KZdQBam7AA$oLea>*v|xz$m-sPr2wt?|$A#yo92>-V3eg=JTnU zOiDoYUBXmmppraHq2Fw;ve`GZFsr){!W$$iMppusLT^h(c~>MbLj1iGWc`QB{2f1s zCG^Xp9#M*u8AHRb1v5PdEFPmgd^a+wd7*qaa;fDZ3Kj%j)R}7|!WfBeqivRnQrqb7 zS5`q^Xj*#l7CpKuGp2C8mtMhwu#1o9+H9kaL}P3|j_leLQ+`X+!FYGf%UdgF8ym%x zwb(7YvREOt?7%#&{MGCxH_Thav=^G(G5>U4d)TwJ@0~3!fqvg#)}y<=RN9x^vAJt- zvu8PRp!jeDaqlrktph*e@h2KxBiUI1QSfPm`=+29qr#jqP-Tu9uELwXW>6Tk<|-_X zff7Rb(gviX1$(>Awhcuz#5XAV6q}w$dA|5q>Y=Lltjamrcu~yzTT30btd42euReQ= zLd>Z$r>wX%g)0#){q16_LQKw?cb;FBWOB*8tL0;Cr9~LCtb04HG~SyL8t*O0evMH* zNaxoc=+Th%Xs3D2|3J>sP*C|UDjnMh3KkO^Cq#?>rC#Me>lI|-Xt$3%0kaZn!^`=nG$7nj1iii7sg`Ba8Y}R zDz5hReNmd8hwzwlWx?c2Jp7jz2#d1n>kdu#3V<%atmNrAegQ9Z`qGV^cf|CvonOq+ zJLZDj*@Lv&x9!qfg)BB>pp^F}}+82#C|0*Q^^bfCdp59t^ zXNy!!^{wS!ws6F}-oGp_?zzILGRLe-8%#1yPMUuf(<(8EFmFGv)oP-*ci8uYWVh|% zYkDp(s^;W;;^xtzY3}Oohc3{4`68iOr?xuoI}v$i{kwP22HhY6NDfZZ09V~Q7 zXZRmuk(jUYA??v2RL5;sc(8Hbo6E-)^2?CRE*mRI#MJIzx@(K3XxW!{%13zO=OyEn zk8X-zmRwg>s*;c?z^do;qM|h2%d5Eh@tTv#HEe2Vjby8hLv~huc93{{&(@Q4iVxlWSdRv`-_67()B15|c9G zeBU~mk)2Wl|+UPqG%sB4Sj7bU;HbJQ_@b#jBl5vh2ED zT)~-XJ^RQk=57xB5x#M;> zUMjiz&aw7*j^x)4ZjV;IFgaD#eqU=NG84&c$t8ZW1j%j5bKztwQr)w#<-}atQH+bt z-iJOFxe{GXsZ4@xF>eOeMPY{%FrB=tW7ekswI8OT`}HwY--nkO?~}}1w`*&dq?@M zlM5Y=U%skpl3n%T{%U6v5xj~elX&1dR!_8QFR%Fi}GC!z0Tp# z;t|(S*|d?w#s*aii$46aS9%k2OUy?KeaQ{oL9KL$NNvoGi~r(~FEO6oH?TRg}V zLuazy4t>S{MaA#iv{tTMfpb<==m?&pp5SW*hxeNvt~~E#%)x&HE^sdV%O2 zYFKGgaNVPq=sieW28&A!@yK~xTzed_)<*vdxsgxnE39%l*hLgSvZy-1?D&)#A3!2ILe)hA5ytD07< z-eMw=Tau(LU2{4L z$KT5cyB|gd317N@rCP>)2yx|i+#Rp(PJVgEL+Ywga@m&;E3}uGq*i@ea56uT9Vu_g zCLXj9scgw{A*is4J*MVpe<|^iU**U8j}Uf+b2_N}O@eA8D+vRi3HGDdq6HRMN1?6T zNJ=ADBrM*$Ui{bK`m%&D&>~GCDZM$>N!GbG`u3kFC)vEp?ckTyCI8L+$9#m0$QLph zn`aHlr~3-_CYM?-ailwuMpBZ3{9sQ#qeGOs#c+tNz{PcFmm7nrBzCalz|>Xop-XP2 z`(w(0K8-$ziOMu7jCE=B<{M`Kna3%|w3ac1@(`A*Vt;83gHG})s7mG`KLy@UfixZ2 z6*%qzQQ>_=UP%%nTlxr3#(e^F@~B&Sqe67YAq=|&#Z7+gFOW@v5xhpKae2tkn+lhy z$6+R1hCp3`LJ$3=2TtkX8-LEzV~i#9ej_vGZ|+w62xVC@#iM}Ls4#X=8#zYfypw1G z_^@ZR!lFCB^gEJ#d(@8>K4V`l!DkKOk>i-egzGs;xX9ChB}=PAB4}j3IUxcMoZjDw zz=S~}`Ei~Oge(kBY1B{lZ=y(}1|dA#GsvI|E% zO{do!ZlV#fhe+>~ht`BmUR%bGv-5AHdyhhkAqALqpA?w<&UA(a@P8wRn10GZSPrbx z?vteJIVg?h z{)&VDdeI(iPQSZi0pqzlPWd5O``%c5`X$y~`$$r~z|h}_K)WZuF(N+DN4sz$w6u6n z3TTY@D_w{mY9Ku^Da?;=(*J4=JaD8Ze+4fSUy_Gia#MrJk@5Ka3#7nJ9(E}7kzWS~ z9IO(?d(_H;39sFrtl=c)aq% zT+uz_VIpi_hT`$Ei?41<{*|*U#IcJ?oZh2;tngJV#B9|)Cj6cB!}t4Co4@WGT8tY#;>2l7t%8OSPPMgPiZ zK}covfkjm(u9#>-ZDQ!$Gw()Df&fOk`!6l5c25wjR-Ay)#nsqj0s z>Q z`{|On+(|?u@DA@&)yYhw>xp0#WU{}{>-@^WvV9bPZZ*jV6uxEOC-J=#2vdYy%Xg`$ z5$o9Z{ypx@_c299O(y?gt8>x z2}T63K9=VDCfk98dl;C_6vAl(n2q_o8&4D=>jJ%0pipm&zo^DNLUsmDCNL({p>K2( zr49#?$>&|)BLb4_Fx4sA25aci27iVQh=u=xV&g6KljYorO@vnKhJNRN%KK}w1dgG8CX<4p zNivCY$87!qfA3)#LoBKZPGG>Y%Xj=CJ;?{N{qHRYn$;B2$*O}ntWe|!DV*4FP)|&K zmq_yWR&#I@M*86!7f$N9+7Osg6KXS&UzLQ(?oUvXto=2XpcW35xp<7{#lOBypKTQ`VzaTyE=dTyk96f zj7Mm_ChMq|xub*r@JO_;?nS${r)D62{i*?%%Uj|6Yi#cJJq`dNpcT{}l}ll0>^ z#LAAl)E#g#aikJIYMWF9=zG86p4^DAj9nb3gz>a^Cz=wHY;J3u_Pe>0| z2PQq=ZMpbLrG8Rj95sHE(j-6eWHvBq0Q%59kkbXKG9?g002*sbz;p(X5W}P&Qaat0 z=)*P%xlG?z#?qKn3KyZJ(wGkyu13o^G0iDdK=b1HastwkbK;cP%(xT943~u43B8jm zU;8#liOWnlae?8M&tfGG7G3U`4-)p@7YorWX>6%DM>Izodk4-N&FsW>1*eK!j%UdU zbVeo$87>UpCiFlqa_#mYB~CNJL^;EC;n6}GH_XM7t-W#yv#>fr-&cobAs1S*?+nsF zF0tgS4&q17$FsRJUnvwv&vIfF4Ut>4_IzLpyBS}il%ZC*w-CGhBK`BhDctditBB>4 z`i!9vDR58gf%$OJ3Eb1-iIFyse<-coZoL~P{D0N^Q3Vc#_dK6 zq;bx{8KJp5Su}%|AXV@j3OsAK2P(0f@g_lnRf>6Kx$aBRtKIy>f)I+1GgY0 z(6gPGtqL{O&A1Xp43)!o3zf*-U3)$#h0{zZah~DE@XN21Bdanxy9Pauc zfv!Q9@3(ru`I(dNko=as)h7p#tavWXlh(KcXm%%d0?trAe3MX;oa0)tpcD=>fkYX@ zwc(LM3*{a4la)*`Sr|fNVSU4r?@HuEXIb)B2RuMZTMB9hoI#4>*_}A7a68fb#$tgfY-T)(;)YAZ9fexu z+SiH&r*N5xBq|(_{BU0dgP}>+5$nI-74z>^*M3@quP)XDdS1Y9zM~dh9v&2*hhVt@U~MBc zgXIQ*wF4UlmY9Mz1~Pz&DPY?-eZg01?S-!=$^vGltblEE09f=10(M2rV9|%bf%Uwc zU`-!`2`tf=zTks`@}m&33I>b5D8T3w28_NUu;^P47=2uT(FfS^&IfGzrZ4#Fp!{4t zof|Ov0H=f$0i%x%Ecy}vJ6=)1=nDjE`b+_vzUd3TqZZeopyME5^zi{kA7IfQ1HABpDD8Y_ocT6d^vH8_I?qbhk(@X42VZ71*Zfn zg`|Wig{Fimg{6ckg{OooMWjURNgS%oq*4s&p@Q_Vz;mC{58o8NEqr&lUASYoTex?4V0dJBbok|G3nMEs zsY64tqorBwn=OEZ}b_TDLH(Y3_ zY`Da5siB6UmZ7fUTElgQ8x6O#h?KNi`1@1K$u5GJLLR*I&Z0$@{D+Z(mb@KEU8jIE zXho;Me55O$&kCvH6huU`x)&u1C5j}9C5k6XCQ2pFO_WKLOPrUuAW^|B;zO?rR?olH zCpWAf$=)fTVmm7>R0GYK7IFeDqlq)Zi)t3`MXz)UU4ovKQ|OLo*0wd{GUGPmG2=Dk zGvhZCFcUNrG7~lvF%!)*JUSfPD2wgQCltG&S=_^7(L6hgrSPoPMXN0Z=OE{Ia$iBJ z;pb=;i~oWAA5zN`qHKm6h_)*&nK8wiEZI|S`8$~p+e)W}@uP(YgSVh1I{B~I@;Ze$ zqoD>c6H77u?fU2S-|CAQY&SS>@YX=gaJ%7o!?%X2W_$cwGXbte0=O2)XKVy@UsSS% zzi;Ot0!=^>_;=#E&>}kv9W7aXaJwxzJ8W5Z09(INux!@Lxf_5 zl7t=zl?c5PY84s~Vi%qzyii!<+-9+hHp6%#3Yp8unUfY!Z@Vxp@Ew|CFyH}Npt_g? z&*f93fae_yeSv1rDbmD44T6e`dE^(#Z<2SEkCeYFUoPJ+&ogh)yiM~Q=Ur4%#rEDK zFmNsmXCSE9m&-U=PbUCDYpDx^6+_DfqD4{lbWV4QD-i9L#n1#h2OEJ{9X)r7ES`uZ zg0P0u6JT_T3IgFyNdWnj_W-Jq9Y8gD0#J=20IHEFKs8bXs79s$)o2<5j6sXFfsBP3 zz^AkUs76?j-pCf98WjOlqY!{<#0yZ3Bmk<>Gz1ufUQU91?n;19=?YMd8bPWNNN)u3 zCt#%RDL^$k15l0T092!C2rv|?wGyJPoB*Fv5TF{#0#qXeKs7=E^hOMju~8vFH3|Z# zM$-^rSZr?%1bvXc4J6?~fg~v)iN%b-rjo^)o(O{&Pa{P3K{5*zNHze{gM;+CGXk4R zW_fxd49f8^pzIS+w+qr$g5*yiInRv1rc&*ho(LNvWl||n3jGv7YNrN}?_&XSbOy*| z>5RapSfrVT0z+VXJ5Y2~E(1#$Q9%Q#Mt%HfQQ+Z&2T~;)?onyP=z=LVj{qz+qrjD# zQ2y04x_o8g|B(&^MTM$D8>{~K!aFy)~_I}kv!ftk|j6!sTE zYV==TIvv~MGW`GsZ{+do9GoegPBCFOQ#zf(wniw>orVB|k~~;wBM-=Y9d@R4I)&Yv zP~Q8$ymZ=X0quvIh5&;$@*p6eJ5xHHV#Z>obUHOsHw^&>m9^kKY&HP1h%;r_DQ3NA z1onR;Fo^M>P@RJ_W!R~Tf2M9dHMTzu5eDaY`k?gYOc{2HoM%R0|M>!g_OX$mXlten zJH;%|jKKc$1;(J3lNiH{dOD-4BNGRtkuM5V8zt0s+E$wBOBw`IYVH77YNi1!H4lR; zHPipRY|Xg1M0>p*As|cXQPKtHUE79u7AV-wTPloa=s#NRD5X=IDcUHj``dnHo3Van z9z6deTsa0D7Wbfuj@jxWnemR@!VQh%ZJX%<^P| zc+tVYiSApU-!`$2w{2Q6V+tqwf)CpYj`sy`K`VB$PxNFDhW=?^@ZUe>x4vK?(jV^& z27(I6GsAtsv(R(CcGrs!28?&tv-uS5{o3>I`dhS)_XVpUSAOmKU((4u-u2Ht7@Uus zIE$d+w_X3^#qFJt(VRGqU^mpOKXw{{XeYN%;Y53P{%=nsn4SQG@}OAg&fww%D7Z4< zWL%3b#iB!z)ym1E%0J z0-KrxpN0s7_qw$~E*x$-1BF+()&UByD3A^bE?R+7@EL(kg`!VSghBnhZ4l*z&(Hvc zefSIwP?Qti2nI@z5}+y{u$Ts*4;kHQ2r8z5cN&7yZ}^ZFP%+Ik1lXiJC@8pM0YYrF z9R(CD;O+`guM*CK&IoMmti0c6dB!vC(@bE{#)nKOI{>$4gSs=2Atk&a1C*%-8EuX3 zRRE=vAaet7zdk7a2AOA=h5&;Y4;fNQkkKjJF%C+NKt{TaAk_#Y2zuhF#qMXd5H1Af$mJ{bc);1XG*72Q=@4JFqD656lD8E@c z(<%CsR^%a@oX&+2Ti7-we!lSDeMhavbD{7Ark(liQ4t_XInq&Wq(a#XU z8Xxkzv#4uyBsDxJTkz{h>amtt_|Tx=j0A^TyvJV=G~_oexPEL@HZ;In=If~Jv6fkQ zHuCRBWsf)WLPLHnxiQ6~BfO!(;<{f)c#rpFL+6wHeuVdUGcP>kSHwMRbhJ1$tJw!P zFho+7;oYFUP- zNJA66p_b+GDbnK|@$iK4-%OF7lmrF!lnGEs0PHEDkO06_es_>?EYz3{&&r18r$dd| z6SK0%rnZWs_V`KFDz+Y@6V&0k?wnsIsE#A}cMKfxrUftDF(o6q(pG6c~Jb33P(_#Hr(d z$)84<#b~oWD7?b^kU$Y(SAhO#MqpE9`_oWh@IKFBXrJf!k=#J+E4-}+6kg$7BB1ya zyblQ!=a~`MpGTBq7IFF^4BqESgU+&r``|#?>hW>9Kqx0jyAKz0Kv7PRJ_N270*h%t zrUu|@AvCxUNK7;R00wuqz-AvV=74%ikP1G2S~3td0_l$6Vh$)Z0_oZi0M!Vno1A_C zgBcGEf_@1=t1khN@z{W5JOC}?jKHSKc%~o1;AURh*JfULnC{;PyJO9~@G#xKE@Du} z8eM_#{qg2hAjTRs*60dmt*=9V|2~V(@SPYK{P$~p8_fInDbDc*TUKUGy}ji;=~9yAZdn~(&^OH2xKX08rJ6Fvex*i z{GhDxg+2|diqJqND1Sa9us_BtNP$g0 zm4BuTi|6>`U6&vW1t63AKojyHt2EP4V3W?M1%>^Y(rM@t^sJo1KiE4ymH_$hE}f1w zCxG;AfB-h>jM~4?mX19oNClrMolZ@SW@__)yt5Q$JY>k~%#>lLNH}K%_Ma~>sF}Bt zL1nlwP~-s%B#>aY^y@I)zt2ndeI2Iz_oe2sv!20}nnAAhZOHH6m%zu)jRy0K^sfVh z|GHO^4zSeB@CQ}4hA7WR$nMXSPG?G|@;vhv&D%83ao$CxZ!?O4Si@-vFqrZTLwlG1 zP3e@LyF{CCZtvRcWxgs&1ga$b_3~s*$iTy@eTsMDIViY9p+u2Hu|)Ai$waBdxrs7~a*6X27bGgUg?~7rf~lwTx>K^D z*6N7{=Cm+tBx@(n99w7(vnD(z*#`%W_wMAkvYpe(dP+ZhQ~0*<-QjlOj^S?M-r<4a zk>Szdm!HW;7N>W*Q`GRpN@$1E*LGYP_mli+6{lcm>;Rq(YTf8$hKEA0gf|@sF6NP6 zB)>`CQ9e@su6((C`&T%HDGB6e5r|l1F2V|GKbw^ns)2?&(Y^8Pnz#(KFg!L~6fO#I z!h@U};5b=Y|yN8^CXOLe(LI)n`!XxAdL(|dxgD2O?8!j|dHe6!3)KJ4v%TU*Ft>HSu zjfPuVgi4xsx?^Y-or4FVU33$z=$3*Q90z`uW)U(s6^Co$ywKPCm?E%nMH-fa)r2w7VjKHSI2Bx9Fu=%mF9vE5_K>Lb% z0uW>K0Xh*`fQrNj2k0r+1GE+hJAmFz7@$d?!gvIBgVY9P%;AwY8hGBs=i()5GOe*=OTHZRtO-pXhs zGI}XJJK;lA|2{j9=yYj)OehAD?JNMfRu{ngA_82C1a2+=y83dj+%O=O5$b(RY@drD zkWBNimxoGQy$p~lOK(Nj&X*Wn_{^2R%vw_iclzUrg9wl{8)kqeRuH6##Q`+2tNY=iR#Z@*46|9_;64W)Zr zp&c|cRoN+SIhqmJf4;!{eN-BuROd`pc8Xh$W(4-1FEBXTt%IW7X;>FTG?rtBNQ^In zsG(!S6a<5wATyYZ8p0sx!%?V_uttU-E!l=qL1Iwb5cFacwI0QA#nPJ*)G7yRcWx^= z0cAn!#(Zd#W#mw?r312`7(HavP<;Y}jG`4|=~qyUWGtg9vjWSYH8PrA{mC{c+Q$Tc z=wAjIkzZ>gGsvL#UQDJ|B4~|@)QiwHY!LM({tPUA5z327@~1^Hx)L(((&<%LS}vmP z34_{Kqn3plM)g$6e#wkt;@_r<4O-s71_7CqMHtZQabuCdS0Y#l_U|NN5Msq|bclXHO?J4_CLt#}3)s zA3vsM<7V%4Ol|ZJ%Tzs`omAmJpQ+r;Q=~4$(In~_rtTAy!z=9Y{tQ?BCSBZFV_Cd{ zd_+`3gOeDq>YkU>=PRl9w5y8lin2zB7qsZG8X4`J_%G%Iw$VzKKP znyr@aK%?t}6 zgn3}redrM(Is8HhqGLihe#qY0UTyUMm%>l}!h+yZCnrP;q_cj=FRaN|#+YSY*&(}o zM|)2D@_Ri=g?JWC2YP{|kF)RQ!~^r~ZCE}o4_m$*ceWxGJDXiF_|%GnhY#6U3JHm~ z*p`3Lpbi{kcrl+m8y6Dy>2zANsBpJq-xrb0{rbyT&TEURJPwUGU&13Vth8KQ!fS2Z zhH?!kbA!;kz-A(P;L!Y z%7V4Xr){BL@`pJeynUqtg4V2Cx`%K~oZgUnk0V7&AkQ^Zw`M@wGpJG8F)y!nx3zz6 zi4W`J4r?{v+#vGq#|V)Qr-QZ9j~hQ6EcQIO5<^T2s<|F~yVE})$GNf5k2o8e-+LIzlGOJG7<;OS5r7c>c8ctM%4ESF3^f6Gswd841-p4n$y-R1y zvORUnj(DsU^UinQ%Fm?T`yZ)VtWm&_Vu%g-DJZY=M%EXi|@GC_V8&kAnhiu*K z_nRMgb8tC$4AwNi9ThGFbs=gXJ?OeQ&)aj-_t!c;E5ET;ZC!xOdjnH8gN%hNzOk78 zkEG33o6cT7@7E^~mW|2Y{2+?+W~wyX_Lna0?}%9$I3C4V;>rP%O{0Y1iI6AhE$?^X z->%1v+@rjjw=FhBeC5EJmYOX=c1*=R)m~`s+DFJ$N!S~%uQ;1TIGL6`%Ck}>%_)ve zS7}a6T0qgbW0*=e8qzfC!oIaWb@%Wc1BoCeg{u$bl#*&S-#&Q3O(ik)NboO@^Sv}* z+V82njQ4ewz^appO~I{!nYRLwM|^U2T$WQ`b!(@*xZ#73SrWMF%g5x@?)H`)c85bL05SMMgo{%VJN?erB-MlRse@DBO|yUBq%`4gmNx6`bd!xD%RIpqzi97j-lutRU&ou)Wfdc{vg@APRb5%l zncSKfSBuJenBvK?$?aeOIppbqjHvZVIvNfI@8d-Mw$2khD|fzm!cE6X86laNN zUcRa%j<9o(S(OyFtM$P(y9h_$OSAK@?%SK!&r^}o`C!p5^^g7b=eCUOcV|FGo0Dso zlks9H7AB@6oWH^r|2VdedpX&Uf{M7i-sOtAn15hCYp);g#>@Bb1l&bka?jp)Sb*zUr>7a*R!ZBYb+qx?!4AUf@h(1r@^llSC3-ROU zR)ui-8!H~Pz-?l?b!>y=IvM4s2gSOV2 ztvMu))9NlgCC^!7B4YG58QTUQFSYtd-YoI_dWF;K_ItRU&)h|6T-aKFEAW8rCZqi^ zuc}s=QRY}S)}H3hS(#CXK5^ivSP@0^ec@n-=w^mUL@;+Pd)2~MERC^dibBbvyY6N6 z9v9tgxya&j=tfF@)s?1_ZN8t5nEP!g;<~0HbBa;05DTIe} z1Ja*0i2Gi=zSYvOeZYCPIZr1wGFCEsJ zrWA`3jy~skvyORQ%SL%BpND_@#;Yd|ebKY|N{O+5=vgmyyI`*QA-#pT} z$2B+QR;o2$n$6s&=J&-33$g@MUK2Ie=7!P>xM@DsDQl6hLYYVM=%?O~EHv8JI!} zlV7Rv)fF<_$@?4jIavjn&M(NgaLFg>(!Kbm)Q>OxiyNrWyEbe&v%RkWv8l(l;k|@& zzOUr83|>>^G21UbBx*f47Lj@^_<7g+WVADV!wRZL<+b9Bk|Z;$%Nv&eGu!31c|*#( zxYWXoo7}Z2w99*ww!~;WNiX615V!P#@W+0_sjQ5iFZX*)YI%6nvO?1z-&`yt`9P$V zU+Uwt&IU8Gq&#wPmG!VjXDNrN#Xp1mS=tNFsuC2CPgpiT?QvDM5POrlrpncmfl1qV zk$qdh&^^8E?88+>t;Ez+Li`HFyLxgxIEw*$5$0;AkOf)Ry$YM!?5>=kFIcm$mu%lb%v0apo!3Sp>yavi|*Zd zbg1STUevC6{r)Q}sD6qk4U=}QXfLnS(vxZ=X1rc>x6sRBi&3FR4#E_C<&w#v<~o>w}vbC~jUtl)ikT(Mn+ z(|AZk?$A1kqf}#!n9w?|J6DR(>*Tc_Tl&977V&>rb1z`9;Cw8};nD2o`zQ*5;!c+hUXfsT>yoz3-P!Y0?yuUku=qirl==xH)t9$Co;tlcbaQL=>N z9DE0s#+li)XvEJa>E~GAYqpOChv8#^y+*{D^S{{xbT5I`$?L9BI z(kZ4D3^B6-fpx0W%|%yyyl7%a&fB079Vt%lKc+cZ}8>I z_biTEO%fB^CBR}$+|st2U5!E<& zmCc=xzEbXEmTp|3Cve&F=}r!7+2g0Op7&|bPM*Cr-aazkOvvLl(>+;M`a;6WEfybd zX4-HbKYl}Bq;ei-JAR(>(1rEc&sOoZ*oV4p3^m+W>FOD{sy%PyUE;vb^(!jHyb8@D z@91(K%U)H z$Q+!#%MIv-sZRMH1V)Wv@JP+3Gp(M8fN* zva@U=7HoD^=do0vFF-VeKV5tSml}=IvCJR!&$nD}$YntfqV(G!%gVsj<+#1eahvU4 zN9>PT!^&H-o#4o_6b`bAqigIE_0DhG z8FBwhtxJ5%FFBu4t(M%NWtAl?o5kYXdMoQxwBJh`4szOcuAPDI;+t~}?6SYS*k91M zVI5D-_ACV>h8)`tekr*wo%g|SRVBk6=7<*=JP2lw>^b78xai>g_p8E}a9%OFh7OJW zr(~Z;!?ufjk8AD*#`#7VK8+X~GRF4{=E{9?d?U+_}zg|1;($u34STm9xwx?L*<6?NvfZU5)a5@f=5 zjz>L#N!Jd#tf|F1(kk*=*V?qa)TD91N@`}_S%rXPKbsv^Qf3x0rf;!6r=3F(JM?#) z5DX=ucl7-Siv0Jx*?PbVWA6J=_%NSj2RLOq4t;va!??rS6BAh?Dt)D*J z*5*a~0C_K#nk>^NYT{p};G`&Aqh4*bV2Pqv+~Z~6Ag)*Jou|Ke$W zTxYUjf-UX04-wLrNp|it._2.map(item=>item.targetType)) should be (List("varchar(128)","varchar(128)", "varchar(128)","varchar(128)","varchar(128)","varchar(128)","varchar(128)","int","varchar(128)","int","decimal(10, 4)", "decimal(10, 4)","varchar(128)","timestamp","timestamp","varchar(128)", "varchar(128)","int","varchar(128)","timestamp","timestamp","varchar(128)","varchar(128)", "varchar(128)","varchar(128)","timestamp","timestamp")) + } + + it should "create change" in { + val filePath = this + .getClass + .getClassLoader + .getResource("copyOfOds.xlsx") + .getPath + createTableList(filePath).head._2 should be ("create table t_order\n(\n order_sn varchar(128) primary key,\n product_code varchar(128),\n product_name varchar(128),\n product_version varchar(128),\n product_status varchar(128),\n user_code varchar(128),\n user_name varchar(128),\n user_age int,\n user_address varchar(128),\n product_count int,\n price decimal(10, 4),\n discount decimal(10, 4),\n order_status varchar(128),\n order_create_time timestamp,\n order_update_time timestamp auto_increment\n);") + createTableList(filePath)(1)._2 should be ("create table t_user\n(\n user_code varchar(128),\n user_name varchar(128),\n user_age int,\n user_address varchar(128),\n create_time timestamp,\n update_time timestamp auto_increment\n);") + createTableList(filePath)(2)._2 should be ("create table t_product\n(\n product_code varchar(128),\n product_name varchar(128),\n product_version varchar(128),\n product_status varchar(128),\n create_time timestamp,\n update_time timestamp auto_increment\n);") + } +} diff --git a/spark/src/main/scala/com/github/sharpdata/sharpetl/spark/cli/Command.scala b/spark/src/main/scala/com/github/sharpdata/sharpetl/spark/cli/Command.scala index a0d0b37..b9181a6 100644 --- a/spark/src/main/scala/com/github/sharpdata/sharpetl/spark/cli/Command.scala +++ b/spark/src/main/scala/com/github/sharpdata/sharpetl/spark/cli/Command.scala @@ -1,6 +1,6 @@ package com.github.sharpdata.sharpetl.spark.cli -import com.github.sharpdata.sharpetl.modeling.cli.{GenerateDwdStepCommand, GenerateSqlFiles} +import com.github.sharpdata.sharpetl.modeling.cli.{GenerateDwdStepCommand, GenerateSqlAutomateGenerateFiles, GenerateSqlFiles} import com.github.sharpdata.sharpetl.spark.utils.JavaVersionChecker import com.github.sharpdata.sharpetl.core.api.WfEvalResult.throwFirstException import com.github.sharpdata.sharpetl.core.api.{LogDrivenInterpreter, WfEvalResult} @@ -93,7 +93,8 @@ class BatchSparkJobCommand extends BatchJobCommand { classOf[BatchSparkJobCommand], classOf[GenerateSqlFiles], classOf[EncryptionCommand], - classOf[GenerateDwdStepCommand] + classOf[GenerateDwdStepCommand], + classOf[GenerateSqlAutomateGenerateFiles] ) ) class Command extends Runnable { From 2ba5bd4119bcf6b747aeb38153937ee8f7213c11 Mon Sep 17 00:00:00 2001 From: li chen Date: Mon, 22 Aug 2022 12:34:44 +0800 Subject: [PATCH 2/2] change function --- .../src/main/resources/dbToMiddleLevel.xlsx | Bin 17934 -> 30080 bytes .../sharpetl/modeling/cli/Command.scala | 17 +-- .../modeling/excel/model/OdsTable.scala | 1 + .../excel/parser/OdsTableParser.scala | 1 + .../formatConversion/model/OdsTable.scala | 28 ----- .../parser/createSqlParser.scala | 116 +++++++++--------- .../excel/parser/tableParserSpec.scala | 16 +-- .../sharpetl/spark/cli/Command.scala | 1 + 8 files changed, 67 insertions(+), 113 deletions(-) delete mode 100644 data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/model/OdsTable.scala diff --git a/data-modeling/src/main/resources/dbToMiddleLevel.xlsx b/data-modeling/src/main/resources/dbToMiddleLevel.xlsx index 2d2ceb1c5f33d318e25cf71e882f6c739f591acf..2693b01d9e666c87f791450d78de7d1b12ff23d6 100644 GIT binary patch delta 17176 zcma)D1z1#D*CwR9yIZdh#-}l_{ zQTNO_Yp-)=-o4gd?|M&6GgnoKcGY%vD*ZG2Qf1Lu(TO$NV0geXpqyhHJ3lCTY zfepdR@;{D(vp!tof3sx!AC?fT?Emv9$NxOa`9F_xJsy4h<}#5ffsyc-Ogv;XRO$<5 z#fJtOhimgt1tNVs)d~siKk=m%PQaorsL95JNGL1?Nz-oG$J;1?!|*iVaKJNoYQR2u zCJZIwSMUOzHSW;5)-W=U76#*3aynjmH^_K~g8ABti#ZYhPgi(rUP;m)dKHz>yO=4b zM9?c;yZ6GAGQz!FPjQ00Jgz)%_N=z&8H@L|Ji&4g{3wHQ!)$dog z(`AZkSA}$3;ZNG$9$KaBlaSx1<@q>%88j2D+pkzR>-l7crg0^{Ssc(d(GCyrkF}Ke zrZ|3nTXb!KubA4NI^3KiRczfFzrH#2+{Wv3m#13po~o;;xn`_$BV3HeF4buuh7m^4 znhtH1vTAZ%3HrM506Njmv}i%0*`9D(Z!PzypfLuw@GN$iC=3-Lj_L{6#1SG|MMUAR z*nCn{2%t|PC&I#*l!qpRVyFOo6M11wDnrAdG2cXDSh+G>V+bGqt`V8tSD{{wRs2A9 zNPYw@6L9Dj?C)mbon`ugOW-r1nIou;ScsLztMd~;Jcxz_a1U5=9KRaNi74n<3MY;I zA;ciPIk1*zh}_TZ6BF%sD(zi!*Txl zcse|KDQMWlyrF2|bRL%=>%#`15oj74!ASB`{1EW2>Yj8RpezRioBq|I?+^kEEDRA0 z?9Wi}>@gMy<2(nV3c~}(kVgMN^>aXyUTh92ZaARp4rasuwbfx)AuuCD-s6ZG_(Gpu zn!e~2_`%u=Gapg$1ncNBc>8e=!IhxNjgv{DC#k6FOH09Yn06_f5d__1w)@M9uXFR4 zxuWht&tC#ZgfMo6HdiJoaC~*pvjva}&9{?fIpM(2lS9dPzakZ#Co|$8(!Bs+@skpW zD1S=RhUzBOh6uMh1fZfjql7p%U;=4k^dxQ0!O?O>We1X}>^EuNWDpYOXgdux*u?O* z#JNEr4#EKiCwwqpt@)y}QeN1zPYp^Icokr2d=*UYB4q2pa(6Kvk&hk&?)(HomI(II zG&V9fxV|H(#LE-vy1JKU0Ye4o7DBm!Bm^y5#l6%in7)oTGX=>C`S9Ajz zry>9WlvqL*?M#awy9t&Oi}k>~;iD@^etg)Zg)yz5-|!r}T-waTO$nd>de?oP*_aqr z^M2D5VaFG0F1ub;$m={AyLj4m%?~f-oI8MH-K#x{-%rU;6n?8{RZ0oqmM+k@wAw;F zoLPdTT0wk93I#(1XNVbF4uKCfApnVl#Bsp5)`_k=LDvV1k7qTlmmNe^ofQxH#z71R zy4i`L3d$Yp^}@ugu<1Z`eU@4jnwxc&+(32J7Pn+JmCAZOZ#iL~ zGtu5=J^Ieio@?hqb+eVLoVwN*5|Ox8G%M%M{X|^!-1hlP_*tRCZAY?umqVE9{d04m@R0kzR*|3AtC*qOZS?J|-LT(em)A5j^83lAsKr^0W`pK)kl zX(?1r*E{?cm8L89?iL9gV+ed;F0J^~|Gg{z7=hEZmqngut|)e^?zI*A4RIM;CeG)9 z;rpNaTgFGe)rutojNTGjcQ}c68#fDQy?q6XD!RV#%0r1ACHp_gVa@7u7yyYF?FET9 z79XMHY|mBvp1CkvrP^Z9LlTBh1?EX?HH3yWMszhSkmrmG@U!TCW)z=bE7Q1j(K^@) z>mLIfh>{1P^PF;{l%fc^Ixv&Hm>~5M-(>CuQI)zM@dXNela@g%j=2h8w;gf2@eCzf z*DV@mN^(P|@a8oIJ@Yy{IpFdfiHjXB;pL?z<~<2q7`P~MRIU^Akt6*mV&qZ;t5z;S z{JAQ{O0Ullio1^bcLlGr&gpG{z15Hsy&;lA!%5B6?B`L@%9qgn1uH-3c4S_*U8b@I zeEm^#eUf@9KQoWx-=#5nKx#V{Fj4>PtS#MPN=!%S^vp4ShQBVHW(qJ^dcR3S5kphc z*k#%J_Qp}F1=l|oOXD^*^L)SPCv`W_s+_>KtU2#%t>TPamr|xxE&Ocjb-BK(uZBdC zX6(Kb9wC8k!K^UU9(|@B)E6uba5bp3=i2mxSQIX#X=w?>Tx_$jUX@~Pc{?JJLFo38 z6$H*j&y-Cq<#hWn?PLIU!@XIE)QTzKG6IV~r;Q3GHavvFeA%k$*h4b96e0-^5MOpW z;qr6O^7b~TvA}3^1S=qN1!;eUuuo{IqOCQCBlFtphq5`b3-!Yidp5+LjL4Z|M(`rk zv5D_YLE*jKR0R2|XrQ0k6wLxG`8yE-&!I94kAYUQG`?r-Sm0-XKx#a)g;lb50xb9} z-W1-FT;8yfv4Ir$HvOGkzOdo3jS**o7Kor}jDEcGg+{W@HfF28?tz?h;r5nhmjn%2 z^p3RM(Aj8rT`L^?Saf<{tU+^7q($3bu^?hQkMkfN2Mpq66ezMgeXDowWr=xacPnfr z6&!O|xaxtpodxhQgG8ph;S3&oY$;JIu!gtD>nMdOLaUhU8&IMvWtP z$wPvHEg}8)6$zBXx%_i2ay+iZ#NRT6msMGnqK60DTvM9-h0`c$SuQz}5z2kQCyzqW z1rFbct9FzM62EssMB0akuj1Zv%aJHTu23LtQ`vu!17(OAnLS_C5BcJpj)Rjz=(|E# zD`zio2z1au8NzCFvf}N$1(>uxTKMK$N0}VJN@=5!^sZDDv~UXgOD^0k-lXa;@4eT+Wg7?R6c|NwNX3dtsl_vGJ^Y8P|{C- z*@LOLpH$WGVM-q(XTR^dluVcZDid)W$qbPG0NQOFN5-gaHWK*Fh|js!iR{f8pluoz zP^{Ig&uIwffTdbyeFzF~qaxNOw^BKlm7E<#=a;w3(X)nSUeRb@YQ1L&WBFB$h>v}9!@1TTTkuJh5@>m z2$gCpE43)OD_I)9i>@{v&)fhc6@>z1yjzIL;lY&XFuDUjeMvT;@1l~RiDO=JTlRS`#XCPzO$3Su!lkH# zM2r@gj^X*i8_ul8ztxQ3n5Tp}ncP#cx|m&~+~A$OfmBk_yv5f3{LkBDuhbUBCCV+X zAq~2jqn6So=Clq?OO3YAzH|gqIJEn9Esm)@li&8-m|y1tG7Cc(&Vd=lQMsQCOSCEk z)!F7e-b5{$PFZha$!Ipuz<;!nN2*&eFY;5_t%@^uWmFv!u;vJ88%I8us#_Nw#?sL( z(mhO-DUNDSpq75&En^PD(E<5R| zc`mu7`!n@T0LeDOvyH|fBc|3qqMuOR_gZlI#9+^{DBT$PO{t2C)T8N5EpiRX(=@cs zr0R5vOo5hpKU8?nt0qG3?-*R&U$*$HU38BhxUbX(H|Tih8+d+Gq8>jVZJT1+Sdne3 zGi-M}#$WQ0;9hGU|E|qld_`Sh5V&u*#l*X%?dlDpKl zJ+DnEZ&h2U^%PKpS>#_Y*%+ROXVzZTM&%AJNI}G9H!Af+Tw)6!EKgcRNqT3@!QjLI zYa#Lt36QGv2_p2O6P=ucB*!?&ROAY~wlfQ=c7<0o2b)B}o6zXt)K{Is*!Z<^*%Gja zL48tIWRg7)1HrJ2R`DXQcj8u6UJvRbSimxNTRk|8hqAYruy}@~+%f)MZyoRP)OR`) z+tBukmIyii4v}ARkq`(GOdLe?aBHCeyWE|&Z**bjAs)RmFj%n=c`VWy*b6SZl6alS z*$DL1zs6F;bS_!MST=*0sOk!QuWOQ3CF~jm9m0rb^a?s_F3_@}VF#B`zzY%@9a(%$ z9L?w>3@!wOU$x#tqv2pQ3lu1d2BhGk zk7ptVk%=2AhPVg86~PQTNV|kSo~r^Omed)-0>`>$@{BAJBikt(wvq~-06px5G(szx z;7RDaxOj0gJ9wHXqk3^;v1}H69KpM?3E!h-O!T4XP-qx~%} zRg*wGQ4OH-8^HBX$;igG^w)AlPZbSvwx`3e7FnZ8p~myo@$CuxO2qC|B-U6Etc-V) z$J(MRk*Z1NluL^T;NB;L(Ln0C`gY+el}?K#`BK4wp3kNUvu7x+6r^mEho`Its9ZUQ z6>}M*SKR|}6ENYb-l*6Gl|B>kwl#q|dBF^g_-V2Y75x-8#E}aVh!bNgDeE2tt4f-; z@sX%(6X#6^!(+L^4JudmrTM_2H$!CztaxF2PbeaB(Qlhmt@X3L=J%aY)vO#_~43!iV}2C@+8HHO=qx0*GNO?m|f3@T4+lL$9D4 z#sNfio?m@JL!yy$X1^+rtlqK-nXZW#k?RtX>dsi7(|pQB?LjwXR0leK0P!~IZ0jK~ z0$e*I-)uAwQFKL9u@EFiC6;AKy>;e-s`xI+dKz3~8p=}+F0R?x`2{D|?*0pyCFvHH z*|4o3(*uJcuojNlyq{rDLpH!{;vt^OF&(t>j)8Ai%Ud6SD`CyX^Im$z8iw#+Y@S|Krdz% zI!)@ixMLT(Jb2t7aTvcm*97Gx(tpTFKrtK|;0ncG$!M7p2B7Eq|F!eLE%a&J6fXb5 zpz~ z9H*UJRur_D<4T9^p=-2Kr=0)?Puj1NNSdzV_(j#Tpz0Pw`SUJp!6=tOTchY>nKHnf zOIO?^pkbY8u#L3vAW9o*oi-IR+=}QTysojYt3fYnh++VVqHd2WSY3MeVk_ooEBQ@^ z0MS$R9L~$1<_3cshe2&1CRaih?zsDk?DW&Yaji>5i&qYo#(M)pN!QuXH-$5n0~yvi zWjgy`D^N;rf|Xd>${Zcpw2aK%0l{QT9Mtgq5irtXkvz9`dtvrL^aIr#%XI`dvk}29_t<4yM z=#bLHI$5&iyY3L-KBlpZmSpppRu3Rv5iJQ*QVc!2+{(G>Qyar~#F2h7J%2Q|ouVZH zdGdPCLA>v3Z<|WE`t{LJWI_UB*gRtj@D>C@k#4L%qR$9)>$9VHhXkZ%rA1r!bD_kC z1Y{*7+Gd*BeT(C3JXx9kDHjrORNjY&+!Yc~u6lWF8QNq+JPD)KKBw0u(=RB>8UC!_ zzj?jrnId}|o<5;hp0LzUYrI#Sp9{*on2bGFQ!9qnACVnxPTQLeLUug--$E?#ffM>c z^CzqUJda^EY0hDT6zpM>)ph0@a+;tJQ?jf6`FmNhg?Iit4%&Y4z?vUFjrggc8R%2F z<&R$IgL8Y_d0A*UQG!SzM^T>F-GS5Zl56i4LpuO%0t`LE&()I)*E#1xamafH*7X9` zAHr;*Fr-ZroTn_&7cl8o3dA8ip*UT@ef3r_SsqCrO&pAf@?6Sazz(f@sJubIW`6`W z*#oezT%=IWUiXg?$EYwgZt8LNq#MXQ#Mx*W94l)mldDW|#7KWvuZh7B#ZwXbVkQHo z5MGCF$?1Lijjmp+7q)G8H-p8@r+E37TNhleP@0-I0QM2krk?`rLx^mA2hQby=xp4q zKdvkJ487_m(U}4!ucNwz?HM)19824EGL7o#6!kRDCEux{kY{lQm;`ck1p$Z`qJ-JV zYU?LSKVFEKnPfw%V_pi{=X9pPvJ{f(c!n5~V5g#1MACXACs#M+dTN*X?%FeeZQZVZ zP}ffi;3o->)wEh5T$@flaQMIr#ejIBapKpsFj6312urE`i5Gf<#Bszh*O}0$Zivo2 zP!-ul-A?0LNs-`2n1*jR?>+f}#-p+*_RT<5N|!}DjWg)THIIiwrRz$KQzZ}`;UUU4 zz`?majzM7-r|(4-T^N)B9$;r!hy{i(uQH3JnUidf`Gy7l`|!kT>`Uu5%1x0+HQ;oa&mb-iF~FfEP=n-NFQu>bXfsdg1ToBQCcZOlUqwykxh4 zK(PAGrcY3#Lz=sr3YSge0_efNwV+BQ?H4Te%;k=XFJ?yPjLM;v3O=D0K*ip}4%aW^ zF^^5C59ZtkQz@nHpP#X9|76Hf1;f)-l%QKa5-RIVsRM&q^}@XE!(0lPh@&&*s~anB zPwb{Dre&|uQOo0x#`Pa7HAdG58(q1YQ^Q^Ws<6(Jy<(>p{6Z|oJ^_A={3N!#m+D6A zo)yb`cCqfE7s{8=h_ul@=qx^*E!AFF9amQ5-SbjCOV2p%{<_nqNiW>PDVCt(hrfL@ zk~;R_=kwP>>4)Ek_;&iQ6ozAu)|&X}`92GtlPqS+=-KC94IxXdvJ}on)z)2;?RT`Y z*|y*)o%J4e3};|i0<8gG)_m6~SaTWPE8JgSd|udPekQ8@*} znx%`|;MkAYXE7!{GK2RsGk^Hw%_M< z4jB^KGnxS6ENCUEK^oqkxbz#dLYZxfO)@)NNxI7^4-sIDyXhc7xb+#$Rq#yD2xZGQ82|a9OFCFC%BS8RO`6mE_^8!~X@F2np zfw;8n;QT;&S~JdZ41{ck-fWX=hzE0wx7+V)L9Mgr;D5vaZ=5L}Py_=Tz{>KpAzq4k z*n?CU9d~&xli$_hyI0o$hsNg2s)Ys+CuN_TfZ2XM!m*F2U*{Oj{DZ$AAx19sxB%@z z(iX$y#@(euyk*ktkM8cyTeVwa)C=iUMhOyHNW_~_`Wi~Vn$XaTzRdxDGqNOG#04e1 z9>qeL+)DKZa96zs3e*KA2xcI1Mhc@O=-8Xp=oQydd|Z?-u3ycEcLL6ftdoTl;&J$N zM24ERiSIc)DxIQDx~NtPskRTqC*fE~VdVRT_e+EkSAZaLW`90RvYxMxzpUs6*A;I> z(3AgDE=_-TFRnn*vd~h^jhbKJntxMSkG!x;;vE7tycJ%`L!RVrfTGCzeM_^G5`v?fUXd|4CLT+>^GQU)%a_p^AEH-y zSCD|oXTxqq{>04s$C3O+Jhw72dcrewL(uMZH(mQ3wd#p4za={xeOd8x?EU0|g>)5v z6$-$7&er8Cg_us>)AJCf;~(hzBLrqhUn02on6dRwVfq|tnuPIF7}YyfdzrT*rjQ{> z46!5%mGzVt+f^UB!gFX^dv7r8tV}z2LP5k0zdcLWe8TMefqrNLJ>texBYPRb5mPPb zmg)+Il!rHCrZdnj^#W_(QSAUSgrlYyAqouRKB8@TjlrVFiwKz-a1(pb<1qvPk@g)n zE1aJ1UmR@9hrVitu)pCbTV6=DJ!@LXT9TV`XrbeE>FV21gP-1CX{AstHWWNn?LIwa z2V6Yu7YYZ_+aA~%qJP*4KecwD9{!^QuF?>KTDuRp1}-C|a8<9)L8)l1mze&1_JHRr zD;V_$Jk$RhJhMLHIeE?@2gLlpH(CaFUI`o}rYfQ?D&95F78q|BaeIJ$^x#Vk7O7F=2SKF-dqg}61H1Xj<(2X z>>AA=WjtrJY4Lnl<14V^p=Y)kX%P`Mt>D-#ox+vv*G_-&V_h>N$Uws^+NIgv0tw%k_!Rc2!HJ=?eJY|Izn3=CX=T==b zwX6lF5P6A}kmO2{E({NGy?8qso&<<$odZO&3Lw1Ty^~Z@B^=4lsvEZ&YMJReWcyTK zoPD%1<$4*g2k0`k-Th%2!##33O$f;1eNo;LT~I5dg&?VfZTl6z=c31Rt(%`&g$gHU zAhE?M=5Si1DjeReZ1#IpCM{S(FclOE{8o@Z5<3|j2QrtFz|d({q~<|$@|7vv7al-n z(1u(L3UBJre8=c1Qo67U#2_5!SZMDQPf!qstRk!e`5RZgYB;2HX%}*%_5B%)F83+k z_#g~9MMU|y_A*B~uBsrQv?7FjtM&%vNy7QQ3?kuXF_(LgLYC)57M>PHzfVas^%q)n zRHLxSv0(g(^1-(V(om4yCm#ESX$b%%HY!NfPk}}o`+AO|5amN}k)*>QgUd*Ql*8}? z+4>+=uLB_(!Mkomkc3mvA&OoJlZo%n@qGZNRg)%7M7JWR?t~Bp9ddyv$`WRQCU?@n zz^{Z<&w~Xc+B4#I^Kz>@_X1R@@GjS0%(a=tCStC6b`! z@19m%vzRQ}P>A6Jucc#0;gA~Z@&mhyu}%xapiR}(j6bXMfY%6z@Z>-`jZFft0Jc@$ zWHCPusH8F!@A3?OLOm>18ge-uQAZF9gN^dPa?c+tp7pO4zxmsWUqK@O{I6F$+tVg) z^kKz^J*@aD$Ifl$Nc3V*t>LDn@vX)KwB%W1>@X%9NFXE0+La}0;8xSWT@Saz)HDfI zP4cReA-uFVs68==(8L>mz30CV%3*E|ZbU`WiNFLcxBURs-Q|K6E+iZh7o{Q>j;sVd z&x&c|LvOYPHOWmtFo{*({hV{xtI62)B_NE5%yNqMuJKCj3(2VALpWP!gpgwo2}u%f zO?KTPVsa~)w@ijduR~fVX5m(~K~)_jMd6~6oUbWT zpfpD{WHU@#YJnQj4GxUO=PAyX!=Q@e#r@6T@awD4?u9)7^y{WGO|8%Gl|d?vO^4)x~_E&B_wi?)2{%A3!DK9}H4c#+~{k{GdD6QDXzMql3 zcKg}ekG7RW2zZ-4=N~I+f3L9$dDBC+hBcqD(j-tXy7oOplcJ($F4Jd}mv@XVbu*@3 zXfPH(5&_(jgp3J=eV{hq(s7ChYcn!^@tv&VQr+%-m)9q}w&gjE@6wnu+`-&2R(Yv2 z%YHM|b2{gtpb}dRutB7^uLIiQ4blpXvMbwVHP(nzARsgDg;J*WLqih?qf}S6Q4-3X zaBy+VqMB5NAvyL-LKBGC*?}0Jt_Q{k*9RSdaGSRYpN4A(A@IS#9sl~X?o5PAvekR?&F5AB;;i2GboV; zBK*gy0_ijUeEqSk*g(q)7wC+h%EiL$)&k)o3L+(m6{wjDr!(`_N_b0KP>u7C~iq2cT=@^joBvJ)ygKG#<6n4O>N zZ;q*Gy5|qqApnc^2ia7tHQP&Q!52iD!Xe=~Pi{cPI?taf%2 z(WzK6m^w(#1ZdIl>-3(BADR~dYBV&^xNl6)*FWFN_FHoo4!%fNw0wzXndfV*t$FEI zc*rHU36w<+V@Q*eq59VNX{y3=u06n!eyp^7FhTaS&4N6-#&-oD=t!FJ$OCH3fOpFA zi|je%Tq1PW5ros z?>QJp{n0`F&Ic@aIQCk1QK*auFE#4cCfqT%L!7Im{LB*6DiM2VmFV!Ef0%fEe?jZ& zuG@0ceTed7_@#$}Z|6d3tFiv&_KJMwUdg4RFP;$n`Fy}td;qRn#?LSQYpE%Ody-fy zTTtNT0N}8_-(Z^b*1frP&D4Ms8%EEm;T}h5$dG{5K1Slq(s=u!f7U)`N;`~7NB`HO zbGK`xC>dN2j26Vh{O_3DrxqGTK(#tkcjMb_k|kA7gH-MPoPkldEuw`F{_F8HtES}5 zluWMnC;|pVCmhb5>w$etYleyBUCZ*~(p6SnfGBW!?IKc|Z(jW9aPZjJl|q63U-l3n z+KD3l;4c_gjlf`96U#YbpM?@AI~c8vK_8VWy_>vFck&K(v3$#4TKk4VZD|@2IS2 z?*Q|wT3;YE3E3P^5kp*E=cb|h4FU-jXZ$F8R!0ntFmQ20f>xb;#A3~G(|Hl?&>>A( zPjsrX@CiQ3(`7^=+EGL1u)%7xPx|^&_Uy`A8hyjV{m}KQe74Du?NBup)!z=h27k5| zy5E+?gz=(n{J4v~DQMKe=sPr-q3`>3DK@}CvuPA$+D4}|GWL&bOWSeIh=awHolXlr zj(jl(*7+Z^mV$W{YDDwu5t8nMoZ-`4{@bp6u0*;G)>^k5N37{8H`3up`f-k5?5f2m z_&3QRn_o9lV35@i&D%xvd+?aB-GsLM^?pzRdx9VN`inibD&ONNfPRlb`v|)&M@w?E zYO%C~i9zJ115uA7fu7wFcPm#S{28)KT>p{981X^*2(GCQtVAwT;7hZsgPE}gt|M+% z>=`Wi7>jO#2AA=OiLFN27+jy=DEL^b3lM03Tx2S;jKigU4Lwb7oGtW6Pa{al@F9U0 zy7tJvoLIMkkau7HEc$-a8tZ-aSzl3-PP0b|37uQh8i_?($3qyg8I2c(dj8>a2XVE8c0L>C_7zYa!E8iJ%>iJIWk--#MZ5mpdU z^Kk{mL*GQqMB-cc(DLgtSDvbx0u1^#NVJ4Ao2?$}Ro~q1)~wGKTlaV8d?~k?>V?mh zhv=)%#>H=xt{V;p`xG7ER5GE}UA7z(=b6FdyyS2b)Y<_wiFf9sP-3z|QAlyVAe~vN zHe5HUG`Lvp8FOf!S7_IaZPXCL04GIav+y&z_{r0)q@yiR~hnL{q*=gLnbk_+#>MuQFNPc+a(=)ZO*>?`4K!=?x7R9X3K7>Wk;V zyrH8j3?6i5zP1%j{qNI{QzIHojP`5|7sB`1of$!V;?Nc>NN1*#)2uG!b)AfDQ*zmD zji2%p1*9`GGX zaA^Gcq()0P#+`R2U~dpPTo}-53v&{{ADT0#z42j0iHMaTk}-y+s<3?hIK_6&&S zHK;sHJT(Qz>g<%4`H0PL^kz3zM&ta5ICm6P`5J~kEVS@TgwJ?~#OsL_;OC^0UGurH zc<3u`vJS=hX7KLBV7e8jH~vI8imPyiVOE3s+F(!AjFI;fdEOnmO=#mmjdpO_o3O_n z&{*;4O1!`Y!3tC+hIwa+i`v7ZqF$%<_3g}MUY&7dCDC`N z7wbTfI?I57zx>FgZ-WRL!#?OiNmO-2YRw=PSk+F{vY!;jbi~Fd=cy)V-OeAS%VJPP zp69yx=}bz3)HBOZ>X|Z-dIsY`J#)Z{2QgX!!TLj*JiNEwo*I8{2c(|K-ecJWsb_{A zCE5g7si`H8b@r~4`8E9`FIfT@8fxAlR5r?X03wF&*D4?K2s0yrZ1OI zo}V%9IZuC5Sn@!9ZwoF}(R*al5j6IyoKht%euQ{}Hf{pf+=cBBTjS_UI8fF_A|Txvbj->h<*svaGIm(?8q$UJyo;q`p$WUKZFB zX4_7E26%|fUyfN^%t0WJ2hlbJe~N$qLv;BE$~hlVE=KYnC=Ucuw*G6BKlSC4Rvgk; zF#F#d9S1flMPK6t!+vfq);%t^St>W!pwwiMte2T=cu!UY`OfUqQS>)|Apox+O#BN% zo!x`TxVKvLIAbqn$gsAd%DCt+f6jbka3(5J*2qfGGTRD3-xb_ksfNViZ=w?C#RsW1 zTQrIlp8~w=O8`*zzWM>@Bp{sYqN=-5YeKAiGXddz5rYitlI6-rfz^Ax(KuHXq!Ob7 zsl*m1bwN1))cvRuE8G5ml^E+T1Xai%M5OIrisTo*J{ZQLn*$fTk$^3(Y}xVXl*ms$ zs{R4ZmRreJusydfdVTuATcm)WXp;9*!Nh@1q44nrzYvkGuQzof{II$ zJkI^M__seQUEDV^xtDh=O6HN`EIY57`y6GFs~bG#x$ai|W&~R?A=M0T7Y!N(F}PKo zFM#PWf#qs(oHnxOA;|!%5A{Cd?+1eB02?qAG25>T`#b@?G`Y%|Z~;!@A3-o@YPZck zXq+cKKN7rTtDK2H(K)rE&r$&c)o3fiKgZC|83H)66{Wka?Tji#zQWxS145DPN`>Aa zA#v5d=z`@Zi+t+LTe5a9zeTL9>gv}-TfORckcFNN>tZUUUE?|I?uH)X;9{NaJ?kBU zP16R@NNMpITUUMraKB@WOrFx-G05+#(X$gk21(Bx9EY=h(hTV9p}%b6^h)8_(X+(g4!NLx<9L*BHrOUlATWw9e{x<-^A25%A!wp#FUg?cXY^ zLO#CDf5dtXC6fi@qZINDD>Wd=z+r&}aX_h(AJ%7&Sd&J__F32=1Y;$F9en@K3mBL zg?`;FR1LP_s4p~?7Mo_1kS2DFB3ON@+p+q{)bZZI4 zkV6Sk%k6Ufh^oYQL5>DEjc%#hyJ@xYrU|Y`<0Dm&Y87uxGc)QcG&b@RI~EA4JHnm8 z#($$?b?0HxN@Brxk!lRMbg(C}w&7sbubi&qROFg8^0cfMDTS&?#9V4F%D=^26R)lz z>|M99Po!}?eOq#reyqFe!M5sN!zQx5gX;N_R_4=DcOgC|-`i3E*d6RyUCFxXaxxB} zqt$8uAhx!q&J~jOqj3N%grTjQs-4TwRexs4LVAwjP&Q@i3QG#G&`z0)n&7OIY5Fo6 z&(S~lg1_V=4^dq;D(B}Iz-?t46@v3i<{-6cH}PF24Gim6pG`%ss*)wfitm=6T-APb zxCE%{uf$%YwsaCmrKccM^KJxKqkmPL6E2~taac2<3fgXu=lKeV?3Fu1A3gDaz`J8F zx_eYKv6_5wp#hX5=3D&RtS)w28do@)WFH-4P%ZRKisJo}o*Y1`e;JJboQCtITjBylv9 zL`b;qR2WL+tfaO)*);!VeZJ8A^UJ(H_5eW#wL7eWND&UF)zLsbI7Wr~7i;464+{X5 z6`s|+3h-b}w}h{*#Z4TW&oP<$DmAE}{OZ@Uxt^cjSJ1v}5u4=B~c2 zmc7@6I4@ZbijrM_Ky1HJ`NX1QxVkmn{@9<*6jF1p z+$+Pq0vMe8>P!J8>C}R4Z=84ziOFwqx&9nJ>m#Vu(r>0rU{ff}ctfYpW-l`Unekm` z)08-fWC+ofWa6}R7j-YL+6Cm~6;!{q0vm&Q>odbk`)?{;S8s=-7e=7>4@mFQs^R_7 z9XJR|YXvW7BG&k7HU%39Eu50)j8gdXeT5j-L~AxBJzZm{7Hz(KN%f#tflK624tO0N z`xQzn+n~s*!NL9sB2qu=u%BOm`Xbgaf2+lS`n(K)Bn5{M7Su4M34YS!*Q_$Ikq{tc zRNGHMYP0;HHKJJfxpq;99*JH=};&aiQV@SSZ_Me2A?F(U)?-cSB zm@}9G3ZpPO`esN1G1;MMP0$~YIYQJ@SY4lo8FhIRt_rO;7tcdvx}7eExO6Ob3Dq^t zLU($rSlc+^rRgRal34F(;N6E%ljqN6bI_aMqkZ6EWgSD#(?RdQt~r5pVRTdoVr8&- zX;#LrQkNg-iI#&%W)Ihx}laxA;R8 z^d%$xoqhwh@$EbY_d5BB*9L1*HsQbx8o1;EW=x0u5wR|KT&N1neTK0bA8oVh$>B6gnXBKW?a9AfE z$k^s@k_#aEquz9v%;9NR7*UaZYn3Mhf?|lAXem}DA_ADfsT2L@;zQ3=NBLbGO}qlP zf!$a%>jnLY2x*w@R{#n_;r{l(MK6f;kKytHYwvuz(Qd2B+6kw5E}(kTD)thYNi&_U zs!2REofb1=ArbOPeP@QJy=Mo`Y=h-uvN@F#)*H9#s0J-6Z-31%U{N2LNGv0K~&a9 z8H%#pZ_YJfOsqqoKoZaiR!mEIR=^OM?6E`o@6Gs;pUfWH zEdSmt0_Dlct7W{-r=zc(X6e=>V4mjAt(Eyk1CWA*WOvtJ*9#Qp6D06kSW zA%+F)6+(l1coMCwa9^PY_(>ECVh;znMneF4Iue`o;lQsCul+v55$@v*dctUrAaN8X zfCwjev@HLV78H5%{OfZOzk4Pp zd-NPB^5i)~gaD#};?ef=AGW^=seiX^rF^s{6-9%6xVGK-e-odS9V{2O^0*+5f6D{N3J(@zGvU?CA=A-hBD_ zc>CHcph>9yya12{>`w*Q?-LlXf+nDTynK$+9^yx1bdJX>U=@G*?tWF@{BDfH`QRPz zAD8Qc=;m>@ZSIHJKq?(HVXVh*<5v~L?+baK=V8A8vXFmOJy8Azzaahx4-bNg|9A0g z*Y$T75du#x{&CU&+GhRT#S7ubn_&GD%D;A?e)k|I^5o$k-Kh>Q64X9$Y*I zBw29+h*J%qr8wCmgnz9>Fn`0Mt`4wJocz&d6lBw*2iy}UFnEX#psnejJ5!Jt;FlM( zAsCpK9kZvSi>-;HqwNE~`S;lS>y8Wm7gI1W6Jww)FCj2M0`bx58wmo70~0VXLl<*9 zS7xS%fB!z$uNfYc5UkS=^vM*`P}wG=F&a Ee*xc;#sB~S delta 9692 zcmZuXbzIa>(?=uSjdVzNN_U5XG)H&0#L*>vG)O89(%m2>-3?OG(j7106aJph`@Mf= zcV}j6de3(cA}Rs`Sy>JW8Vdjq4-e=oMgxz)QC6^0LO}fs1}mAuREfPJ`A|BoHg!d{ZfZ3lR=DY~ z*z0|+DBic6F$70|@C$0PVIdLU{atOG9y3o z@kU}4PXbAA*JqK}^0;=I89O6I%%6B|?sFNYKlFUI$t}@LJN>5g_7OoJ_)&Ysoc%*b z(ZoHB>7$fC3&36(^e+5z?V{G)Q09&+;$EVR0>h4O<5L9pTES@vka_%_lr7ieso&;W z^ZC%K9@Pu>HpnkuC!RNLVcq6C~fnqno0 z)}%^efvua6zDLu-z?4Dou$NbjVj!~2)@0GJ zL)&gZ?xWOGpBV?YM*@A30}2s)QpoHn@PQ%iQzg&i;!ykO^+o5;*W}5F)%rf}fV&|qXHH<|AYeKDL z^i6?W(tBIePQM-V8<^QiNVy>~7;qi(%b3}#2_yZW7bF%72}hj!BmJTGBo=>oD^WXs z8x+|iqe?ileo%CG{G39=aGfrC&SLZ^()|cfmV<-8MWOlr8xjBrLIOOspEU;V-zr1G zK?iQOL;%y{4ZWz%vuu;#!-A5QY;#%Af--?C5DqK8kus#Rsy5nc3r&_aLNi!TVs38? z&TPFW5Ib09RjKAib{M`k-5q2#b^(>q|CVl`)vUT0V>lC--zcF?yR3P=3D$DoxP*> zfWhosIF5J&Lg3eX&iDv|qEl_YBK8M|*%g@$J=bG(b(7n|oAKoJd7&dyUY0{=!p~-J z_L!qx&Zz_`9JN<~?4Su!Vi%C)@nftq9;tD*H0-msRO*0t7g%lx@g#A%s`6T?SrRSRgqhE0dWb)xWGZX1jab3E-Cl0SqL}!4 zq3Qjz($oiyu^2Sk3VqVzZ(AicC0My@r&-P45NviT{q4~hM8gR|&@MZ{b_E z8$DgFdn;GmlO&S`BmIVH_B8};UTKG$F!RJM<~5aM})IED>XKGr}ck@=Y^zYx31!6|W2Hp+>WEOB3D~ zuPf}yLu)E8-x<=tN-YmWOP_V{BS@EzVKxjU<*n&gW|!d_6T3>aBGsRY=%@Ckg=8|e zJ9QaKwk8h+!m`dr>Q&VnEXT+Y`=^kvwmX?qN0A%*gF=8MTU_own~Qi(3MXh0uPa6< zl1w43D?@-+p`R_D!SN8_9ehOxio!m%5m;!yFHyjOX{b9TTH%3ZFp)S;L^M5)^>VET z4lJ=>VQ)d*PVI#U`^mS|*G&JfAS((rQ;rtGV7x;B-(k` z=OSyV?1#MNZ&_&bgYHuI+XG?~SHVQ{H~&#s6dn9p!DclNg@#K z|CJo+pl|5md1~)Bfm3zkHN^gl;Rl`C_xYgyNf+bT0$r`^*;a-Z&Nzb}W>=7#Lcg&XRm%w6$9?1G!n$S8hM0B|Hc)h7 zr+?6ry|+=g93T(snYlO}e`L$_UGaBO(EjqPK8m|oRX4rTq8O12^>Ld{OQaGnqR7S9)3h)RRfKpV?rJ_0-t4?KiF0?wC_-ZJg|p_0EY($8dQ~a z4Lg*7e*=FIMI~%5DA@(s>wm;(?k9q1-i8<`$tWm^_<|`86hTDRk{&aCLJT6SM$0wu zGZG2JsqV{?__6}{?LUkW!DtSwLw{iq%3#hOz(ry1D|KM>!!>xUyuE_d6sNjDByD0Y zm7+SDae?;*FIgKjiUVsSLW7m7f+(U5@?f0-;oRRG?>1AcB1X zndvx!z|LVP5W9_EAoNSWBV=%vuP=F=3<@ndzIYr6Tl}XuI`9)B-U*~R0dydX7cf=E z80~2~Uq((~C~`fR(3nOF6)kRq6oDojpzYos^IEuyjPkeJBxYc83^e=;Q#R;eG(TR= z5VASC$@O(y0a9L3EJZAcT3SCcxmCsFRro^z5DlfnpH6WDv7B;#3%~WlD4K2%4Hre& zbAN|*SZA=f0SA-Uo!-2stt v7~4aZ91*L<~PFJTUcc5cI{1wk;E~2xE7~TIXD?? zJ7=qfgkvg3y>!QvU4FA`7r4@qJ0+7-xMu?(-0@wo_sIYxq5qEoPy_t+Q2@Y99E9Lq zD+I6}>F~d70v#yvZ-A-(N<2z5KLfn%>j(dqahk48d72i4; z$KA3sH&Y~phdJ)8Yp!}$4~h>DT$Vn$k0N-)&k_w~F=m3X#V#mEig|*UBo(jQNrisOK2;7;Ep# z;;M}9XXenv7^jKRhHS*|NF2*y5yQ-;ApR9meH?eEFg&g6(JL#6$HtqzekSt6MeEEt ztSqMSUg~!eNu{Eyi(OV9%(Yw}_hC;7-Gg@%_NT)os#-^xXD0~zp1e!3MWaIE$4Q$X z?Jhd!3~evb%PuJY+@2aOOy?EM9}4 zD_0}!su~>??aHG(j1RO5L7+M{$K$V?WR*B2z(FOE!ep5J?zp&}qAz$LZ~u#)EW<)i9Sleb#TYeT4Ba5hNE}K`6&=8xR8L z2xE$3DcNsxb3_m)0!PCselM4+0Cbo6z?0B!F%%TRcd5(N-|waQ(XngLgg(=0ak1{g zi0%GG&Z(Pa_j8}Z%@~CLk#q2Kw0TJJUq{FJ;^_Vs?MysOQX%s93#CN|x@rd_HCE}} zbUm>)ac2iJF_V-TXcC@ggDE3YT|_B&KS_&#z42R>dcp2QK@i1-3?40d!EGRcXj6kM zs{#3>;rM#|?HSN}O}Bi!a&L_;9vMH2fP5G4^=gur`IAoF3JL;*$EEz9AjbjdG${2Rq|j&sT3Gy7^V zjp()4nb9S2w#e>W!iT?t-p;-i}3gSR`sbyW!{t+Yq)5*UA>T> zH>dcn&eQzjv0FV(f(L z@e+nhZw<`+;bAzf6u(ti23EAP-)Bh8jDQ{#0ynB1O&PxP0%bOtG9wxke!tn2kf1ns zKpKxVWQ7wdBGs`lrfP_Sh{9GKj_leMAD7cCl3~RYO7^lMu`5EwA8?q+6}AHN30th? zGl+km$nX#RT_rOZ-v6Bts&YU+)J9dhf%LE!+ax7aX}k9cAy@$R6@7aCqbdduY}8tfGtMJ?oLzJ13b#aEL*fDk#~` zimBiGiX{REuo6equ)Tjni#@kM!c-&7Sl`k#DHqnX_v1bd`%kv?p>S3=Q}0#9ZCSCG&a2LE}7 z0tfRep4)vFB=>cOLnXBxB3fLS;v^>dyfU%9h)s$6T4&JX$HugVDD!bXB{}tpueTyW zo^3a^JZuk4weeh090lLUm$-&(4?V~i#_6yzYjbPvKqg)830x=fZ61cb7O8F$YCEwz z94AH91xw2Y!*<8OG=Y7hL|xTl9jIsXQjALo(v9I# zq?GUv2vWT*3kGa=3S_TuecWgm-6}uLKFv+K?eh_!YDIeE;B z=(nv>$~d#@5IE-yBCAKyC{dhgoS z8=HELmG8nGps(`rE`7PJ)T}zOi{;klm86dS2I>J-=3<0^d8`lE)x!^+?D=sU71bv1 z-9GNNM&C12S~9jUTi#{&%y4!HpMKQgj_j?L`G7bag<@y*)xh&ew=r~B&h^d4wgIQE zdq}Ob@>?K`mwJh(gi<^}f-GDVHt2%1)ek=w8a70Or*-XWw%h!|KpX>~Xr=QN7EL>K zHmCzXHaWe$^A$EtH+98R11$hkizjwXeAd|1zD#a9HVBhmF+v^^JR#@*ji(}r{ardf z8^mcvE0g93L%O&FhA~ERI{66Z4!5}_~rS7fxcvM4OaY#~i>#`hl zWUVsb517#o0Q8D2ri)r7tTvl(6zowf2)WT$!+*>82khui07NUQxvEBha;-byGb$Q6 zAxU;QY_|0)2tIPsPTI5gKxEN5QAlUmc4_%6%}8_kct#pwIG9zu&>`-LWY`L8O$a^) zfRrD;&kU7S`B#`R3y~YOWj(8<%4>fD%xvy5MyAjxL`Vu2CyU51w86&c`dMi-(Ed8b zY$$wC6OKpM=(-1O5Aik;nIIR9>2{_>)en@x*%`$~sFiuYVO;+@90MSiy>1nCtuGlL z5Pn&&dke)_{AsnmWFOzeeWK#PTfo5{`xH)DfWV0M+Yl=K^o;g#h0jZ`k5LFToIO$4 zR$UBw4x`TLD(Ebq!YCY60N{TLfY1Krn+*cEgyG+T6D5fIkB|BBq^EdK^^9eR3cGfc zBud$WyeB=?RU0yS3ZJqJiRP%k*a^X6fUmgKIMM*ayd+XRz2&V!l>_|BWO90TUL1(t z(bK%l12^95j}Pb0bw>h)(z-F>GtTQE)twcym1#y?5=W`R6OODzLQwvs4>mO0ii+e$ z1n9R#>EoPZN9cIcnm~-Fh|E};##1pv9#V2(o&=uW0WrX?&P0*KW0^&(7=qkfB0h@Q zZBsA9VjteVWDq9FtJsv+j3!Po2JS3*LU44A4(*4XOn2y&`Gp%n!_1rIfui@B-b?|! z)_0PV#hB~E-P>>Jy&tFi)Xa|2CF(X|!Jr1D#1A{sv=2fU2vC6x@5NVj7GHnb2=2D+ zN~UqtBA;uNY7Mm>B2-J@^hVFHPR_U+fzvv6UEd4xByeY3aVsK(uXF)=-S=n4)t;;^ z?j;@*{vuiW?&M(7I4O9Jssyxg`1@VBZx4w<^E<3><+e=k66!#b zL?h!*g^cF7``=-%TsN=j_kK!8-B%Dlbhu9w26W8Pe54FFw(8JUa#kJMQ*5;Y@0Z>L zt46=Bv?nIw+mks4VE55N4zVj+XVuqr*YiWc)H&ujnP>)uXm+OOM%gPHo^4c-yW<@ZTe0A~mAq@}CH)9x(x^}MvKfcRsp;RJ!??9G;IN8=a)a!JC`m7GO8;%R(PwvVW@qgUapR58* z#X$*t}mX z=r#s?k`nG+?oB+@H??b$Dr-~ybF}7e8wDAN@?|gBqKm^oe0@3BLy8F=I^4`ZKIHSo zvn{&@oD62~IB#t@XwA+9OI%Bb%pZ)43l?uA|Hx@gjA#3_XL>UwDDa-d<{D=AgyW6c zSJ!h&6_1t4K~jees@SDKe2Eb~KDDk} zBuny}jve9VgWlf4>jz%WHXE>t(cE#|emZ*82PgB1meVo!)VOFNX}{E) zBK6}xTIt`-`Vcs$-A}jtL}uqzuNi3jK`%Nxt$w&>qz;mzyJcRQzTF~FFg>)tPvS#0 z0I^*z+=!c~T~^uM;+boE%zMEWdPFcUtzX{|I?{-YD(v%Br;R)5^zNAy4y(^a*&r;? zJaRCq%}k*cZ|zzL=J6xT_jS!gNqvk%4vq&(Ym$BrhQh52Ahi_6NX4#gx%%Ed2US`W|6@l^_2e zW~1|KHYQ0Px1I)eC^td-U&;hW$dTy7pPddD+mT^yeLwZNok<5eKn)`!E!wXJE7M&? z^}zoMPA4R&H!S+j`aNN5BNj;t6`e7<@Rg4dqaVn=b&&$!H@8847h_)6AMMbQ#}t6_ zfI;o|%7*19c{(UdiVXl4@(SD?Sety5H5noJOmThV0)GVLG#F1E)x$JrTyEZBcRM2GHmFP+{fG%sCeVCJfu9wy7TE6~ zDhE;wtlu#>U=_1hh8TMV-jPm=8a3?RF~V1*Q6^4#FQw%YNU%PX2FHf(x7X(`-^81r z-D^I)LxImz{L^5Fl7GCC0xJb@b1yFrN=M0~0PpD7@VRX?_sn=RbhVNAZR>!ybr_#|qopUyGh zmk#)s&H?YGW$p!Yg&>c8KPb?DVKn0OR@t2$Liw}PX@Zw_&#!9tngNMM-MP(Z>e6k; zE}_9`5X7?q`Dh-g00|8MIAi{g0jVP4fP8uWs(Mf}g?gz(m}ydj)J1F;SkSyg9)*wM zUlYtJ<`VdQAw^cMo~iW!P?b_ak{8Cr8(qggB~>eD5ci@oO?Cx51dvMys)cXx@#RGK{5`l^)VGfjw~YIVan+9zAjJy``ws zJ0*9j?DP6L2&C`B&^MM5k1D{eg%FY`R-b)Si_O(pydnhRX$YyCs6XY>aSh*{V8zO^ zG>_9ljJ{1o@Xsh}lUAYmHOrywnkW8&ibcV1Q_{5n*($>rBme`4vE(dCnys%FARx0+ z8Cvdr)W_f=MCd^kF!`E5I0sHF$IMCAS$CD9nG?qk`ZuC(Q9QCpS$16Z>(<2AuP%Dg zn|^-l-x>^A5Z06s0sI$img)-fXIBkT9J6w-rIjfTSDFjlYzUv^&_mEW0H!A~Sj z)zUzPk>G0}59o|PKkhY_hUU;)F4kHI+dR!R+h&KBg3bY?A0vKCkV3@+57FKwS9{kf z1n^DY2NHXnNjYF9qT5AlWP6sPT}28i^4d*gvp@9FqARRme$Fj1R%N3}IVsO%xddQZ zHw4@eHk@?rh2o3FyHlm#l{fe3$D1?b=&7vHgc~u^DH)X{XQW=;j=1SFtIwR(t(~(9 zR&Blc6|8|*5*tH>VX8Gsac&aM!D$y;y@pFn@;zAN7S=7ANnr8s{Sp7~s{Qz&)q z#dY}(d}|YXa$TtZn`u>zYFqpfxVQRbAn;IB8Vi$ShbI*4R}yMGT!?cf!0Oh zHK3q#uivGX)uRtoqKX~n^XRC(dG$>(18Me%>^Msz>rA5<){bjJU^^miIlH)1htG*q z1+qUIJPN0^0>kDgOHOfjJUJ*mrySlgPM3bG6OhL^+rMzPn~coLB(JzKqMd7()@04f zc-h6m?d6oz`XlktY(O=`1XPH#26DieFnYzCJjgM0BvpnzhwO5@b#1$Fd;GDbnq8Sy zJIL~zAC)?u79aD5$<%KT`oejJRP{@TRHu)heKp?jHxt#>M8LuC!W;y@T`b>olh_jd-|@L7G2o zy@~S9swh>5jH87OE*do{wuGAIe`EZVQ!GLVxwOU-nf{}BW7?S8$d7* z4VD_V83Quge1Us^DEC!?xj!Lq3lzW*RVkU#S%^$q*iAgz?K&IW!!;(4{5z1#NX4s;f!}SnpB=p?@VOz3`7X@)T(sM)54)KuFx}7-a4NG+haX>w!IaH0MqrUZa+?|%q=n&>Qe)CKW^4SR4_~r zgIBT(aBw_i4!Wo0f#q(ExBt)C9UmF*<3F8UBJBU3-L>KNi{;%gqI#+UD|V4$`DV7e znb&jDFHANhyEGmBQdO9Q6XwxPQz0d~xV#v5Wp^kHqhz?qi+|0tGQ&lb%;=gR&jqZQ zq@&1nmc~%l`J~FoescW4ksJq2{LWvdl$63#xH4IPaQPCJI+KksIFNS-U-55&lb@4FQP-`0pQ2I?M>cCnmHn7xfJDnEn^``3vs;3q~;eFOKqG zaN>VpG18|$vZ($2c#rqjI{^TofdKZCz`=jcZl^27=I>ERy+jAVrb|2%CF#5p!t#103N3}%W5ak$;#=j7;TH%m96Z~Bf{^!g*;A2Z1FsJn&`l)w)U;j)> z_(Fe3fdKwwjr6kmzsqv}g@XM^)X`*+FUpR`vb&$MxL zFko|=KO3LVJoS26LjlDLk8l6t@pqo}PadU|FZ18n{FxucmJsqg%?oWb3j$c#776|N z4E_$Y@W6-GBw(N_3_j2QcY7SiU9Vr3fWi2}X2}-kWoOM!gpjyQ;5jF# R03dn#eT4@A-U&Rn{ttvwd2Ijy diff --git a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/cli/Command.scala b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/cli/Command.scala index 5535091..7f6733b 100644 --- a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/cli/Command.scala +++ b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/cli/Command.scala @@ -4,7 +4,7 @@ import com.github.sharpdata.sharpetl.modeling.excel.parser.DwdTableParser import com.github.sharpdata.sharpetl.core.cli.{BatchJobCommand, CommonCommand} import com.github.sharpdata.sharpetl.core.util.IOUtil.getFullPath import com.github.sharpdata.sharpetl.core.util.{ETLLogger, IOUtil} -import com.github.sharpdata.sharpetl.modeling.formatConversion.createSqlParser.createTableList +import com.github.sharpdata.sharpetl.modeling.formatConversion.createSqlParser.createTableDDLList import com.github.sharpdata.sharpetl.modeling.sql.gen.DwdWorkflowGen.genWorkflow import picocli.CommandLine @@ -95,8 +95,8 @@ class GenerateDwdStepCommand extends BatchJobCommand { } } -@CommandLine.Command(name = "generate-ods-sql—automate-generate") -class GenerateSqlAutomateGenerateFiles extends CommonCommand { +@CommandLine.Command(name = "generate-ods-ddl") +class GenerateSqlAutomateGenerateFiles extends Runnable { @CommandLine.Option( names = Array("-f", "--file"), description = Array("Excel file path"), @@ -118,18 +118,11 @@ class GenerateSqlAutomateGenerateFiles extends CommonCommand { ) var output: String = _ - override def formatCommand(): Unit = { - commandStr.append(s"--file=$filePath \t") - commandStr.append(s"--output=$output \t") - commandStr.append(s"--help=$helpRequested \t") - super.formatCommand() - } override def run(): Unit = { - loggingJobParameters() - val createSql = createTableList(filePath) + val createSql = createTableDDLList(filePath) createSql.foreach(it => { - val workflowName = s"create_${it._1._2}" + val workflowName = s"create_${it._1}" writeFile(workflowName, it._2) }) } diff --git a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/excel/model/OdsTable.scala b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/excel/model/OdsTable.scala index 5712dcf..1436422 100644 --- a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/excel/model/OdsTable.scala +++ b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/excel/model/OdsTable.scala @@ -60,6 +60,7 @@ object OdsTable { targetTable: String, sourceColumn: String, targetColumn: String, + sourceType: String, extraColumnExpression: String, incrementalColumn: Boolean, primaryKeyColumn: Boolean) diff --git a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/excel/parser/OdsTableParser.scala b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/excel/parser/OdsTableParser.scala index 69a79c3..7d88145 100644 --- a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/excel/parser/OdsTableParser.scala +++ b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/excel/parser/OdsTableParser.scala @@ -85,6 +85,7 @@ object OdsTableParser { targetTable = getStringCellOrNull(TARGET_TABLE, row), sourceColumn = getStringCellOrNull(SOURCE_COLUMN, row), targetColumn = getStringCellOrNull(TARGET_COLUMN, row), + sourceType = getStringCellOrNull(COLUMN_TYPE,row), extraColumnExpression = getStringCellOrNull(EXTRA_COLUMN_EXPRESSION, row), incrementalColumn = getBoolCell(INCREMENTAL_COLUMN, row), primaryKeyColumn = getBoolCell(PRIMARY_COLUMN, row) diff --git a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/model/OdsTable.scala b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/model/OdsTable.scala deleted file mode 100644 index e283b64..0000000 --- a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/model/OdsTable.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.github.sharpdata.sharpetl.modeling.formatConversion.model - -object OdsTable { - - final case class OdsModelingColumnParietal(sourceTable: String, - targetTable: String, - sourceColumn: String, - targetColumn: String, - sourceType: String, - extraColumnExpression: String, - incrementalColumn: Boolean, - primaryKeyColumn: Boolean, - isNullAbel:Boolean - ) - - final case class OdsTypeModelingColumn(sourceTable: String, - targetTable: String, - sourceColumn: String, - targetColumn: String, - sourceType: String, - targetType: String, - extraColumnExpression: String, - incrementalColumn: Boolean, - primaryKeyColumn: Boolean, - isNullAbel:Boolean - ) - -} diff --git a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/parser/createSqlParser.scala b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/parser/createSqlParser.scala index 1f5986c..47abca4 100644 --- a/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/parser/createSqlParser.scala +++ b/data-modeling/src/main/scala/com/github/sharpdata/sharpetl/modeling/formatConversion/parser/createSqlParser.scala @@ -1,97 +1,88 @@ package com.github.sharpdata.sharpetl.modeling.formatConversion import com.github.sharpdata.sharpetl.core.util.ExcelUtil.{getBoolCell, getStringCellOrNull, readHeaders, readSheet} -import com.github.sharpdata.sharpetl.modeling.excel.model.OdsModelingSheetHeader.{COLUMN_TYPE, EXTRA_COLUMN_EXPRESSION, INCREMENTAL_COLUMN, IS_NULLABLE, ODS_MODELING_SHEET_NAME, PRIMARY_COLUMN, SOURCE_COLUMN, TARGET_COLUMN} +import com.github.sharpdata.sharpetl.modeling.excel.model.OdsModelingSheetHeader.{COLUMN_TYPE, EXTRA_COLUMN_EXPRESSION, INCREMENTAL_COLUMN, + ODS_MODELING_SHEET_NAME, PRIMARY_COLUMN, SOURCE_COLUMN, TARGET_COLUMN} +import com.github.sharpdata.sharpetl.modeling.excel.model.OdsTable.OdsModelingColumn import com.github.sharpdata.sharpetl.modeling.excel.model.OdsTableConfigSheetHeader import com.github.sharpdata.sharpetl.modeling.excel.model.OdsTableConfigSheetHeader.{ODS_TABLE_CONFIG_SHEET_NAME, SOURCE_TABLE, TARGET_TABLE, TARGET_TYPE} -import com.github.sharpdata.sharpetl.modeling.formatConversion.model.OdsTable.{OdsModelingColumnParietal, OdsTypeModelingColumn} -import com.github.sharpdata.sharpetl.modeling.formatConversion.model.{OdsTable, TableConfigSheetHeader} +import com.github.sharpdata.sharpetl.modeling.formatConversion.model.TableConfigSheetHeader import org.apache.poi.ss.usermodel.Row -object createSqlParser{ +object createSqlParser { val typeConversionFilePath: String = this .getClass .getClassLoader .getResource("dbToMiddleLevel.xlsx") .getPath - val typeTestPath:String="~/Desktop/dbToMiddleLevel.xlsx" + val typeTestPath: String = "~/Desktop/dbToMiddleLevel.xlsx" - def readDbFieldType(filePath:String): (Map[String, String], Map[String, String]) ={ + def readDbFieldType(filePath: String): (Map[String, String], Map[String, String]) = { val tableConfigSheet = readSheet(filePath, ODS_TABLE_CONFIG_SHEET_NAME) implicit val headersOds: Map[String, Int] = readHeaders(tableConfigSheet.head) - val (sourceDbType,targetDbType) = tableConfigSheet + val (sourceDbType, targetDbType) = tableConfigSheet .tail .map(rowExtractDdType(headersOds)).head - (readTableConfig(sourceDbType) ,readTableConfig(targetDbType)) + (readTableConfig(sourceDbType), readTableConfig(targetDbType)) } - def typeConversion(filePath:String): List[((String,String),Seq[OdsTypeModelingColumn])]= { - val sourceFieldType = getSourceFieldType(filePath) - val dBFieldTypeTuple = readDbFieldType(filePath) - val targetTypeList = sourceFieldType.map(it => { - val sourceTypeList = it.sourceType.split('(') - val middleType: String = dBFieldTypeTuple._1.get(sourceTypeList(0)).head - val targetTypeList = dBFieldTypeTuple._2.filter(it => it._2 == middleType).keys.toList - val targetType = if(targetTypeList.contains(sourceTypeList(0))) sourceTypeList(0) else targetTypeList.head - OdsTypeModelingColumn(it.sourceTable,it.targetTable,it.sourceColumn,it.targetColumn, - it.sourceType,it.sourceType.replace(sourceTypeList(0),targetType),it.extraColumnExpression,it.incrementalColumn,it.primaryKeyColumn,it.isNullAbel)} - ).groupBy(it=>(it.sourceTable,it.targetTable)).toList - targetTypeList - } - - def createTableList(filePath: String): List[((String,String),String)] = { - val odsTypeModelingColumnList = typeConversion(filePath) - odsTypeModelingColumnList.map(it => { - (it._1,createTable(it._1._2, it._2))} - ) - } - - def createTable(targetTable: String, columns: Seq[OdsTable.OdsTypeModelingColumn]): String = { - val primaryKeyField = columns.filter(it => it.primaryKeyColumn) - val primaryKeyString= primaryKeyField.map(_.targetColumn).mkString(",") - createTableAboutPrimaryKey(primaryKeyField.size,primaryKeyString,targetTable ,columns) + def createTableDDLList(filePath: String): List[(String, String)] = { + val sourceFieldTypeList = getSourceFieldType(filePath) + sourceFieldTypeList.map(it => { + val targetTableName = it._1._2 + val odsModelingColumnSeq = it._2 + (targetTableName, createTableDDL(targetTableName, odsModelingColumnSeq, filePath)) + }).toList } - def createTableAboutPrimaryKey(primaryKeyNumber:Int, primaryKeyString:String, targetTable: String, columns: Seq[OdsTable.OdsTypeModelingColumn]): String = { - var sql = s"""create table $targetTable\n(\n""" - sql = sql + " " ++ buildColumnString(primaryKeyNumber,columns) - if(primaryKeyNumber > 1) sql = "primary key(" + primaryKeyString + ")" - sql = sql ++ "\n);" - sql + def createTableDDL(targetTableName:String, odsModelingColumnSeq: Seq[OdsModelingColumn],filePath:String):String= { + val targetColumns = odsModelingColumnSeq.map(_.targetColumn) + val fromSource = odsModelingColumnSeq.map(_.sourceType) + val (sourceToMiddle, middleToTarget) = readDbFieldType(filePath) + val sourceTypeAndLength = fromSource.map(it => { + val sourceTypeList = it.split('(') + val typeLength: String = it.replace(sourceTypeList(0), "") + (sourceTypeList(0), typeLength) + }) + sqlCreate(targetTableName,sourceTypeAndLength, sourceToMiddle, middleToTarget,targetColumns) } - private def buildColumnString(number:Int,columns: Seq[OdsTable.OdsTypeModelingColumn]): String = { - columns - .map(col => {s"""${col.targetColumn} ${col.targetType}""" ++ getExtraContention(number,col)}).mkString(",\n ") + def sqlCreate(targetTableName:String ,sourceTypeAndLength: Seq[(String, String)], sourceToMiddle: Map[String, String], + middleToTarget: Map[String, String],targetColumns:Seq[String]):String = { + val fromSourceType = sourceTypeAndLength.map(_._1) + val fromSourceLength = sourceTypeAndLength.map(_._2) + val targetColWithType = fromSourceType + .map(sourceToMiddle.get(_).head) + .map(middleToTarget.get(_).head) + .zip(fromSourceLength) + .map(it => s"""${it._1}${it._2}""") + .zip(targetColumns) + val columns = targetColWithType.map(it => s"${it._2} ${it._1}").mkString(",\n") + s"""create table $targetTableName( + |=$columns + |)""".stripMargin } - private def getExtraContention(number: Int,column: OdsTable.OdsTypeModelingColumn):String={ - var extractAddition="" - if(column.primaryKeyColumn && number == 1) extractAddition =extractAddition + " primary key" - if(column.isNullAbel) extractAddition =extractAddition + " not null" - if(column.incrementalColumn) extractAddition = extractAddition + " auto_increment" - extractAddition - } - - def getSourceFieldType(filePath: String): Seq[OdsModelingColumnParietal] = { + def getSourceFieldType(filePath: String): Map[(String,String),Seq[OdsModelingColumn]] = { val modelingSheet = readSheet(filePath, ODS_MODELING_SHEET_NAME) implicit val headers: Map[String, Int] = readHeaders(modelingSheet.head) modelingSheet .tail - .map(row => OdsModelingColumnParietal(sourceTable = getStringCellOrNull(SOURCE_TABLE, row), + .map(row => OdsModelingColumn( + sourceTable = getStringCellOrNull(SOURCE_TABLE, row), targetTable = getStringCellOrNull(TARGET_TABLE, row), sourceColumn = getStringCellOrNull(SOURCE_COLUMN, row), targetColumn = getStringCellOrNull(TARGET_COLUMN, row), - sourceType = getStringCellOrNull(COLUMN_TYPE,row), + sourceType = getStringCellOrNull(COLUMN_TYPE, row), extraColumnExpression = getStringCellOrNull(EXTRA_COLUMN_EXPRESSION, row), incrementalColumn = getBoolCell(INCREMENTAL_COLUMN, row), - primaryKeyColumn = getBoolCell(PRIMARY_COLUMN, row), - isNullAbel = getBoolCell(IS_NULLABLE,row))) + primaryKeyColumn = getBoolCell(PRIMARY_COLUMN, row))) + .groupBy(it=>(it.sourceTable,it.targetTable)) } - def readTableConfig(dbName: String):Map[String,String]= { + def readTableConfig(dbName: String): Map[String, String] = { val ModelingSheet = readSheet(typeTestPath, dbName) implicit val headers: Map[String, Int] = readHeaders(ModelingSheet.head) ModelingSheet @@ -99,15 +90,18 @@ object createSqlParser{ .map(rowToColumn).toMap } - private def rowToColumn(implicit headers: Map[String, Int]):Row=>(String,String) = { - row =>( + private def rowToColumn(implicit headers: Map[String, Int]): Row => (String, String) = { + row => + ( getStringCellOrNull(TableConfigSheetHeader.SOURCE, row), getStringCellOrNull(TableConfigSheetHeader.TARGET, row) ) } - private def rowExtractDdType(implicit headers: Map[String, Int]): Row => (String,String) = { - row =>(getStringCellOrNull(OdsTableConfigSheetHeader.SOURCE_TYPE, row), - getStringCellOrNull(OdsTableConfigSheetHeader.TARGET_TYPE, row) + + private def rowExtractDdType(implicit headers: Map[String, Int]): Row => (String, String) = { + row => + (s"""${getStringCellOrNull(OdsTableConfigSheetHeader.SOURCE_TYPE, row)}ToMiddle""", + s"""MiddleTo${getStringCellOrNull(OdsTableConfigSheetHeader.TARGET_TYPE, row)}""" ) } } diff --git a/data-modeling/src/test/scala/com/github/sharpdata/sharpetl/modeling/excel/parser/tableParserSpec.scala b/data-modeling/src/test/scala/com/github/sharpdata/sharpetl/modeling/excel/parser/tableParserSpec.scala index 6278072..ae43506 100644 --- a/data-modeling/src/test/scala/com/github/sharpdata/sharpetl/modeling/excel/parser/tableParserSpec.scala +++ b/data-modeling/src/test/scala/com/github/sharpdata/sharpetl/modeling/excel/parser/tableParserSpec.scala @@ -1,18 +1,10 @@ package com.github.sharpdata.sharpetl.modeling.excel.parser -import com.github.sharpdata.sharpetl.modeling.formatConversion.createSqlParser.{createTable, createTableList, typeConversion, typeConversionFilePath} +import com.github.sharpdata.sharpetl.modeling.formatConversion.createSqlParser.{createTableDDL, createTableDDLList} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should class tableParserSpec extends AnyFlatSpec with should.Matchers { - it should "sql change" in { -val filePath = this - .getClass - .getClassLoader - .getResource("copyOfOds.xlsx") - .getPath - typeConversion(filePath).flatMap(it=>it._2.map(item=>item.targetType)) should be (List("varchar(128)","varchar(128)", "varchar(128)","varchar(128)","varchar(128)","varchar(128)","varchar(128)","int","varchar(128)","int","decimal(10, 4)", "decimal(10, 4)","varchar(128)","timestamp","timestamp","varchar(128)", "varchar(128)","int","varchar(128)","timestamp","timestamp","varchar(128)","varchar(128)", "varchar(128)","varchar(128)","timestamp","timestamp")) - } it should "create change" in { val filePath = this @@ -20,8 +12,8 @@ val filePath = this .getClassLoader .getResource("copyOfOds.xlsx") .getPath - createTableList(filePath).head._2 should be ("create table t_order\n(\n order_sn varchar(128) primary key,\n product_code varchar(128),\n product_name varchar(128),\n product_version varchar(128),\n product_status varchar(128),\n user_code varchar(128),\n user_name varchar(128),\n user_age int,\n user_address varchar(128),\n product_count int,\n price decimal(10, 4),\n discount decimal(10, 4),\n order_status varchar(128),\n order_create_time timestamp,\n order_update_time timestamp auto_increment\n);") - createTableList(filePath)(1)._2 should be ("create table t_user\n(\n user_code varchar(128),\n user_name varchar(128),\n user_age int,\n user_address varchar(128),\n create_time timestamp,\n update_time timestamp auto_increment\n);") - createTableList(filePath)(2)._2 should be ("create table t_product\n(\n product_code varchar(128),\n product_name varchar(128),\n product_version varchar(128),\n product_status varchar(128),\n create_time timestamp,\n update_time timestamp auto_increment\n);") + createTableDDLList(filePath).head._2 should be ("create table t_order(\norder_sn varchar(128),\nproduct_code varchar(128),\nproduct_name varchar(128),\nproduct_version varchar(128),\nproduct_status varchar(128),\nuser_code varchar(128),\nuser_name varchar(128),\nuser_age int,\nuser_address varchar(128),\nproduct_count int,\nprice decimal(10, 4),\ndiscount decimal(10, 4),\norder_status varchar(128),\norder_create_time timestamp,\norder_update_time timestamp\n)") + createTableDDLList(filePath)(1)._2 should be ("create table t_user(\nuser_code varchar(128),\nuser_name varchar(128),\nuser_age int,\nuser_address varchar(128),\ncreate_time timestamp,\nupdate_time timestamp\n)") + createTableDDLList(filePath)(2)._2 should be ("create table t_product(\nproduct_code varchar(128),\nproduct_name varchar(128),\nproduct_version varchar(128),\nproduct_status varchar(128),\ncreate_time timestamp,\nupdate_time timestamp\n)") } } diff --git a/spark/src/main/scala/com/github/sharpdata/sharpetl/spark/cli/Command.scala b/spark/src/main/scala/com/github/sharpdata/sharpetl/spark/cli/Command.scala index b9181a6..3bb163b 100644 --- a/spark/src/main/scala/com/github/sharpdata/sharpetl/spark/cli/Command.scala +++ b/spark/src/main/scala/com/github/sharpdata/sharpetl/spark/cli/Command.scala @@ -95,6 +95,7 @@ class BatchSparkJobCommand extends BatchJobCommand { classOf[EncryptionCommand], classOf[GenerateDwdStepCommand], classOf[GenerateSqlAutomateGenerateFiles] + ) ) class Command extends Runnable {