From b34008a037f10ab3d38c00d82d75c3a3aeb5bab6 Mon Sep 17 00:00:00 2001 From: Steve Taylor Date: Fri, 27 Aug 2021 13:28:22 -0600 Subject: [PATCH 1/9] add connection pool, swagger and exception msgs --- Dockerfile | 14 ++ LICENSE | 201 ++++++++++++++++++++++ __pycache__/main.cpython-38.pyc | Bin 4681 -> 7970 bytes cloudbuild/cloudbuild.yaml | 50 ++++++ main.py | 270 +++++++++++++++++++----------- requirements.txt | 9 +- static/swagger-ui/doc/swagger.yml | 177 ++++++++++++++++++++ 7 files changed, 617 insertions(+), 104 deletions(-) create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 cloudbuild/cloudbuild.yaml create mode 100644 static/swagger-ui/doc/swagger.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..28cd0eed --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM quay.io/ortelius/ms-python-base:flask-1.0 + +ENV DB_HOST localhost +ENV DB_NAME postgres +ENV DB_USER postgres +ENV DB_PASS postgres +ENV DB_POST 5432 + +WORKDIR /app + +COPY main.py /app +COPY requirements.txt /app +RUN pip install -r requirements.txt; \ +python -m pip uninstall -y pip; diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/__pycache__/main.cpython-38.pyc b/__pycache__/main.cpython-38.pyc index ad91191da208381598325bdf1cf630bf130753a3..996b2e6f4ceec9c41b386676ff98cec0bb19db32 100644 GIT binary patch literal 7970 zcmc&(TXP#ncAgssg9}OUF6w6ZB3Y1m)6MZ}Wk(cATdbu?EiaaLlFfLCZju8IFi_7x z60x9?to)KxG3DCleE{?3w>;$6OsZ0;+9$v5L!Ok{b-vRBf*?&-C8@0qP;=(=b-H`b zIp5bkn|*y*1;4LmORjlCQGQ2>-k*uYLtNp1s*1uCruvFacg%Lw!Y(wg* zpR!Z3p4qvl?X*l=wk7YGYSzxmx`v;t=Iwm7&+e=C+x;>xDS{<{;WZLq_s~7AG)d_n-Rro~ZBFnJs*UD?vzQptVGVWJ+fnUM>D$DUJ zmA9NCztVo2<&RX>$NFW>ci4daE-TpY@nQZhe~%5aAx~w)hq`?YEw1xx{9QJ3hKCQ{fz&Ut%cwIc^(*Wu^eH|qQ*W{2k%p1Ybkc9L zcVv2&-;(9;viD^AHcwaXuxso(d;d_g@AA9s1}n*t-{aZJhx{I&gtv&4yaBZ!jc}RKK0#jFHQ^otii)5 z9u9epIol$rI?NNi90g*(q{aGN-HWX?9tI6j=ICX7heuBM!rj^7qTxAPeuLMASBp^B z-1fN7!q|H5`W|y5j=Yq+6^JM{>#hj7DC;yf{4`uCTn4UrT;T{rq(o{(Ybh0-sd73B zrlZkJOfEzq!jd8Ks3)kGIQM97!C71U`R3yKMr>7i7`i(=ETv){m17oV!yF7`OY9tX=M6iNbFcF^oRn+@9IQD8rdB;?Mcaju*{f@Z9$r;Z!x8DhJiN z7xC&;Sv1&8)%9xA_5Jwru3KY1cRCB{&`iDBjw2s1gkD3MtfGlg$WNQY=-iFG^2~#} z8|~gF@ktrJ+WnDE6O_wB&K}{twv^YJT#VREnwL~Tu8y@as0%s#M9@mcIr%n|s#lP4 zJcbLWUU$Pft^^QF#Q=iX$>8yUSxc2Gt z>eE=gZRcg~#?t4DtD755RLiFYP`I4)z^+RUR~=GK#xD zYLLby-wUl{q*T;Mi}c8dQjrOhO&ut$j>)ECLr&d-8@O9;$YY}c%ZjbK8-_0e!SLF! zxTQ35!u@hk-?<&9v7j|LUYurI4z-pFy$Gqu_Ebb&$Q;=N41n7GzyQjE!vkULwc0}r zHY86|v=LQI;a<`a#@&}r@SKZnKX_qh>OtT;q1VK-&YQe!4Dk?`^i{6dDpPEYsZ9G? zv2~`yJdHT_n7e+oJHN}zd#9|L9#5++SMM7r&6P-Lt4GR#+EUvZ?pjN0!|{()RY8t% zq~NYqQcT-3#6zYtgQb3}MP?=Q>npdN2OMi{N zdh40WQ{@-RKf{w4YQ56GM5?BJi4+EJtu(Fm#pT8Mjp8jqFWc5OpESpNU4ZfWRDEw} zs@z}{?vi+7=ka2ndu1+tM$A%6V`25_N+}nc)qPr6aUGd(n1%=g5vTb}UT#D@PL=%t zmSKtDMLB5Hq9m?qt3Qi}Vf0@t%`ZBeD|27WEiKPIT3(DZi!aN(9(h46P9-7Nh>ei@ z+hP*ki`ygyNl+;3i3K_$_UFh7#~>6{Q!V_n@)}YLS^-*fpy$0!qy7rsrRB&OIZm8) zoNB-t2y#xGcbsPp*C(@d9P!icqfS3m%%ETymkd0XD$~vM4B{=!4v&40#M5&6o4ZCQml{9o~PZtkig_)yn3Dcu&CC7{?U+JTf9I7mRsCnvv7WG#K zT8IP8Vwpp-y??1p78xMrAk}sf<>}Sng0gNU{8Z+0wVwgLEs^ z9&eegG+{gR3yD{JqlRWQLCgoWG&aBr#CF&)8zEl9z-QP5yU4&<7`O@pLt$Vg3``{9 zAH+P^O=1{qYUl0O>W+#Tr`i{#XBnNlO1=c&xwMxBpJ4DF3p2PGU2a+AJuRbsh3d2E z=qlyF<5HEkwzX59WWBOZdD6_vo8g&i=+?cNVsysVT(IZyFwP!hsmC_NTX;_> z)T>_YTMh02Mgj)HL=O3J3iL$mqu8RrPuBlNo4Qp}?f%4_oNZTf71sjH!W$4ki?VYz zS@dv`p-vV>;QNhwa^G^xd(b$*$Rn0RwIWx*>L-gYkvUm}(}t7T;(5P4uZHSd4bNxP zPAY)E$zple73d(SbK#=bP7$%7$s!BNd!QA&?qm_10;8*X(U0_xavGVe7`Z$0e$qd} zb%+({JdLIn*4=XV39^A4i0tG18Fb5Kx1HD(yr^-Exi-&6pRK0dw8-u2+X1b<=f_HNhIuz8pn2wbcN30+mDV`A;E&T4dE}Rt2ljZh zB2HF(u?wn;H(va>crB#poEX7(-{Ti+9tCWJ7HkE0Zzkpdm4dJDlSNQ|zSk(I)J*Jx z$_z3MsgQM0p+UAlO1fhXq+7=pK~+ zLdOfHY*Mj8l$)22=-35i^vjH{X;Ae{@`R+Zosku2rYG$wb=+a^3#S@`y&2Ldevm8e zLiW~c1eDQ#+74vaX;YBRy6?X;h>>nak3p=&=?=>;6_Y4|eM=}IC@%B{8V9`y+Fgi8 zByjttN*L4xF_c(e0u(2)XU(m_O#3J|C3i5ffj|+8 zTgZ{1o}lhEs$lY$k#Ix1e+?K%TdsKt_u>u;!7w@5zV8b}DMj2R@ezrikaz+S>tQ6~ z(WRA*#kG|=Fj_DGFyFI6CE0g)0sG;LMRCAPRIayeCErY zms<_bV+L~Rw$LW@A&4A;7if94U)Pk8?+v4%X=*?63aXJd)jTv)9a1%*b?1MgpJ-He zoI1WM+J9Llu?}ia_hx>+1KLD3P62lwBcbb02Hed!(%V@;sc0+%oS0I#u!#OXlsR|k^Mi@&BpUZp2X*a(Y_It5`?3>)P# z54Rh>zyD^er@R(K$8}1V#V={1lJNcwX{Q5>a47F95)@B_3lUo|ZifO(Y+_H1^OZA! z15qdNYO(Hf)YuM0)s1Ah5cjAxAsFTsI z5%(b--@{H)F9$x(M1mJR;b9b~Jb>s>)^LEKbswuC_NaqCR1F=khS-krLp(Gu0!iYg1-6VNI?Ca3I!C6yL-k5 ze_js%>$4pGwPf&;$){K@8Yb5sS6F@=j?em9I__!Ke_(-OkF-p=izO|cXq8cv80|6X z-WKb}p7!5~(f2wt(u2Keu=2#YKLjh!beucWVdaT)4`3X~k*zMCaqi5k%qiy{>UD1N z>x6iV1*qav5|2qNk@y)zb7uj&;f=-O8OlX$t1g0i*r8nHFO&!B<<*KeIusnKGwse& zHQA2n+8edhoI3wm=ePOz#GOSz!>vPwN0!eEmn7);yd{=~taZAAlc(C*WK zzy*oGgRN8#0uOZ%SZ~8^fZnv-)+-}GVC@X@4*idj7iS9xr;%6tvm>vl;^m1NMDy4G zZ{T|l*!Fnm{{cSFfTegwlN2PRPllvLBpM{nIOYq=JL8?_V&w@SikH-8pG1?yuSm2= z5G0BN66c<;)f>^jq42DGN*uI+cpWU%$wf!toCyw|LctS0jLud{Y4L!b_A!Z1NJ!f8 zinPBY@%JQ#Pm2*s{w2g$xIzm8&z`idTkp%<=0N92<~Ux{`5t}4aGYiu2jcYi{sDcK zH1ReXi}xT(4`M9{;~dxn9jwSr25n$)x{bqlxw8_q`LPF%PqC*vyP!;ce+wtjIQEcX z*&aa9bA9?8vp-EoqC_7cQ`__vO3>+WLa6AYhTSJ~@kJ$!K*|K2x(k|+T}TQ#Un?9O z6VW-JoqoZ$9_t`#HiQ?aK`F67 z^|&7!Gi>J^>}|tcoL!{^J`Hejf%?g8SzSBLXx4~17)4eD8 zw4#9<*@Gv=o%4LVf8G=2h8I0b-itjVtCW*W=lE@rZ`qljH+X|Tp`&ZL)uq#1JMS_^ z??iqHlr{xM+35Z9AT5&YFt#2f2gaWOOq6gJ!omhp(m`Mp=_A-y=5ZAaAXbX$RJdK%vl_*o-#`A-3B&js3A0xP!bRM<$qa)VoFzt3K3TvZn~B*q12pP3E3ov_ zo;;nvfovsiw-QvkUf}6|c2e#7f#0nKH7z?yy*n4oF+)_u{38}D2wyDVxhUB92zPm9 z&kU9{@9`@5WzGAVU(tL`^G7vb=X2=ujOOS00{COX>KqqKVp*(+qv9F9_}B?f{J`Kx z_|j(vUlO%PR&Y`*j-M4L`SKp#Ma=V+T~q#7EObusqx>0u>@mF|+ccWTpV02KnyhfM zOvXLcGz;hZNto{z6)EoYMNS&)QWh894UwmP83}Z*U6)))!LP)T3MyCLZ~yT6=FOs- zq*0jcqeaq=+4gyZ@+c!+_|Px*m@F=hbp`s=Ay9sxsZjIg?WB2mAtUCRH}$x zZT7MtT*RHf10oDM@W2!nXWabEzyaCdEpBVx0q<~E^Dg)b_cUMORcxDIc$d>|c2$XP z1RzZ>g4c_&UH{iiE-_=|GGnlUc2~E%uJNJqGvoJo z3y!vj?njV%=0}jQgRi#+%j?}d%;NRCVG`pc1)U+VRf*DcHx_x*Y(4ocw(!Ycu$giJ zr0^qbdW=nv`M^B;Mw12eQD5e%Y;T2;O67ylGqLd^5s_->zI?S2qY{)y91&R~rwA`i;Y`-MEMDxBGFzX`E*IGFfdzJE4SuG!rs} z)kzb*+-ifT(XNpBPPp3Gjulqdjny~UM?0-dcU0lFem=E-7gpl5H<^uIEziPeI)fgd z7ou$({sz-_Z5A7UH;zPluP3nPSxI!m81FT!%68HoCOh12o?u^Lyw?*L#^XGKJ5J@D zI0LVyau#k2$@}VoUd2?p-OuBm$a829pJhv?-3ITj@ot+=8SCjOf<@!r4x$oHyzy@1 z3;NtFp4zy~gP95AVg#SW3SuBVvbf)w*w zF@*+oua+~EgM+H>KyUhVJW2ZoXAhiu9GO*UP<&0RaiOivf`AnB2XUZP2ce*cP0xR_ zh?S{g1`w_0vV@Nt(k3!bQNf!es=Tn^zQy8fJZZyLQOi}R`Z=(|kt#ZQ+AD0hdS2L3 z+Uo%%I))&9iIPWXv9D5BE5DN<*M>6eV!cAjKN#{&G<

{TB8(U(c1e zE7ozg>qucDiTnB4Zhltd7zjTb$v$6)Q}))f2a;0FDct$TAcp0$73MM9^jY0B**sfh zcEx3N(y=9G8q?oX{wYVT`_lf?F#qjUJmzAw#|HJwla#lHZ9b6psOGBU)@cv<#~A!k z8DmB%XxkW>Lw3dZd23`1k!nvMIdM}{OH*4@M^jhRil&~XRZaa9#+FZsZfp+iJtUty zOUfFSY1JM(n8)F@vCHe*OjSs~V(hVz%jd@4&;?f+I^*gPEjl?m9e?C0dtBQykjA`8 z8ml9X8S+0!V^yTFHQ2A}v^}v;7Wh2PJsnv+kl?Bhs|OO?LQAe-OXLzr;g$%L$4P^_ z)R1&+vO(lIBF_^!4e|wB9hhhG!Edg%PzBv=Ty5RFepsk`l@d=G-4LENz7Nev)3~yq zfcEnV#!>>JT%_{^F>sdV*YX{zvRfOQjbDsV0baj$?fpxaf7G1Q?pJsKLRglIw;xTp~*lL*+RdI0v(NJMO`el;H9_blSN}a)G*_GR)Qc z;Bs230Y4${@YqH8jK@541(NxtZPrZ}vTG9ZJ?7&c%uVr6HV!yI4W`Hmv;+JL%;3+; z=&-TL8Q{m1HMRj#lfxB`86!(MkBpHG*Kh!7cW4o;z@h3JxkFd>6kw~tR~_1qsJ3&) z{?O%c7!5$k(2_go5qI9^wyIH2*sgF#&*5=vR4wQIya5-ac`B26+@V);o%*Qyq4AdS z6XS;Q5h~R2oOTK5bY(Zc>&t(@A$(oec|%{6{p-dq! zV?rHBXJG57ewGLg)k&y_5smG_COpdTVh}>PlJZ>|{vHvH;F?CT+#m&cK#5BR1Sf?B zbOtu!{~MZ~N2Nh<nFLU)Gvmn@w5A;A9KezD&^dr3vn9Wc)gUD3ME@P|PY+0+jNaO>?7u#vrduAu z_h`&TBKjoAHxCU77J|(uXv`@lPa=ZrIr|ZS_HYsbo@PJ{X*IpVYqzP$YPSo&-R`Em zpAcVfx9{}BWYSWL@Hw4#h1yAZ3A8J;l50dhAVTSXhR`|7IgpdM^F;j>oO zJ@N(W<80C20MY`l&mfr7iZ)6bwbf6gLk$Fh(+qvSmkdfD`4ZVcEWxKlq{_3WT?y(T=d=j@Uj&@0-<#Py ty_Y)Bd+(Iz@g7AK9qj2COXu<67>lfKL1WcXAWeSein;iSfrIzh{{UY}!*c)t diff --git a/cloudbuild/cloudbuild.yaml b/cloudbuild/cloudbuild.yaml new file mode 100644 index 00000000..5a796152 --- /dev/null +++ b/cloudbuild/cloudbuild.yaml @@ -0,0 +1,50 @@ +steps: + # Get ssh key from Google Secret Manager + - name: gcr.io/cloud-builders/gcloud + id: sshkey + entrypoint: 'bash' + args: [ '-c', 'gcloud secrets versions access latest --secret=ortelius-github > /root/.ssh/id_rsa;chmod 600 /root/.ssh/id_rsa;ssh-keyscan -t rsa github.com > /root/.ssh/known_hosts' ] + volumes: + - name: 'ssh' + path: /root/.ssh + + - name: gcr.io/cloud-builders/docker + id: cloudbuild_sh + entrypoint: 'bash' + args: ['-c', 'ls -A1 | xargs rm -rf;git clone $$COMPONENT_GITURL .;git checkout --track -b $BRANCH_NAME origin/$BRANCH_NAME;env | sed "s/^/export /" >> /workspace/cloudbuild.sh'] + volumes: + - name: 'ssh' + path: /root/.ssh + env: + - 'COMPONENT_GITURL=git@github.com:ortelius/ortelius-ms-compitem-crud.git' + - 'COMPONENT_VERSION=9.0.0' + - 'COMPONENT_DOCKERREPO=quay.io/ortelius/ms-compitem-crud' + - 'BLDDATE=`date`' + - 'IMAGE_TAG="$BRANCH_NAME-v$$COMPONENT_VERSION.$$(git rev-list --count $BRANCH_NAME)-g$SHORT_SHA"' + + # Login to Quay for push. + - name: 'gcr.io/cloud-builders/docker' + id: login + entrypoint: 'bash' + args: ['-c', '. /workspace/cloudbuild.sh;docker login quay.io --username "$$QUAY_USERID" --password $$QUAY_PASSWORD'] + secretEnv: ['QUAY_USERID', 'QUAY_PASSWORD'] + env: + - 'DOCKER_CONFIG=/workspace/docker-config' + + # Build and push deployhub-webadmin.war - quay.io/ortelius/ms-textfile-crud + - name: 'gcr.io/cloud-builders/docker' + id: ortelius + waitFor: [ 'login' ] + entrypoint: 'bash' + args: ["-c", '. /workspace/cloudbuild.sh;docker build --tag $$COMPONENT_DOCKERREPO:$$IMAGE_TAG -f /workspace/Dockerfile .;docker push $$COMPONENT_DOCKERREPO:$$IMAGE_TAG'] + env: + - 'DOCKER_CONFIG=/workspace/docker-config' + +secrets: +- kmsKeyName: projects/eighth-physics-169321/locations/global/keyRings/cli/cryptoKeys/quay + secretEnv: + QUAY_USERID: CiQAW+P1J9UZz+Hr1uonladAW2dKqaiVd5ux8Q9EV81pK0u5V+4SNACcBdnKacvH4QXPamH1N4uJZvZ/0TMwvELgXAAlP0wR2zBw2WhCV82GMiUkW3iGVlbqz7c= +- kmsKeyName: projects/eighth-physics-169321/locations/global/keyRings/cli/cryptoKeys/quay-pw + secretEnv: + QUAY_PASSWORD: CiQAUULEud9Ej8XtwNAb9gkbDVhSGFZYhUGE30fNwR+7ehAOkH8SMgCz6KYeykjgS16RPxgKlrIQL/1TKDt06v4OXGIisFXOkdWC+jvdda8mTzVNCi8sT5g6 + diff --git a/main.py b/main.py index 9deacf38..333063ee 100644 --- a/main.py +++ b/main.py @@ -1,17 +1,47 @@ -import json import os +from collections import OrderedDict +from http import HTTPStatus import psycopg2 -import pybreaker import psycopg2.extras +import pybreaker import requests -from flask import Flask, request +import sqlalchemy.pool as pool +from flask import Flask, request, send_from_directory from flask_restful import Api, Resource -from collections import OrderedDict +from flask_swagger_ui import get_swaggerui_blueprint +from webargs import fields, validate +from webargs.flaskparser import abort, parser + + +#pylint: disable=unused-argument +@parser.error_handler +def handle_request_parsing_error(err, req, schema, *, error_status_code, error_headers): + abort(HTTPStatus.BAD_REQUEST, errors=err.messages) + # Init Flask app = Flask(__name__) api = Api(app) +app.url_map.strict_slashes = False + + +@app.route('/static/') +def send_static(path): + return send_from_directory('static', path) + + +# swagger config +SWAGGER_URL = '/swagger' +API_URL = '/static/swagger.yml' +SWAGGERUI_BLUEPRINT = get_swaggerui_blueprint( + SWAGGER_URL, + API_URL, + config={ + 'app_name': "ortelius-ms-compitem-crud" + } +) +app.register_blueprint(SWAGGERUI_BLUEPRINT, url_prefix=SWAGGER_URL) # Init db connection db_host = os.getenv("DB_HOST", "localhost") @@ -21,32 +51,69 @@ db_port = os.getenv("DB_PORT", "5432") validateuser_url = os.getenv("VALIDATEUSER_URL", "http://localhost:5000") +# connection pool config +conn_pool_size = int(os.getenv("POOL_SIZE", "3")) +conn_pool_max_overflow = int(os.getenv("POOL_MAX_OVERFLOW", "2")) +conn_pool_timeout = float(os.getenv("POOL_TIMEOUT", "30.0")) conn_circuit_breaker = pybreaker.CircuitBreaker( fail_max=1, - reset_timeout=10, + reset_timeout=10 ) + @conn_circuit_breaker def create_conn(): conn = psycopg2.connect(host=db_host, database=db_name, user=db_user, password=db_pass, port=db_port) return conn + +# connection pool init +mypool = pool.QueuePool(create_conn, max_overflow=conn_pool_max_overflow, pool_size=conn_pool_size, timeout=conn_pool_timeout) + +# health check endpoint + + +class HealthCheck(Resource): + def get(self): + try: + conn = mypool.connect() + cursor = conn.cursor() + cursor.execute('SELECT 1') + conn.close() + if cursor.rowcount > 0: + return ({"status": 'UP', "service_name": 'ortelius-ms-dep-pkg-cud'}), HTTPStatus.OK + return ({"status": 'DOWN'}), HTTPStatus.SERVICE_UNAVAILABLE + + except Exception as err: + print(err) + return ({"status": 'DOWN'}), HTTPStatus.SERVICE_UNAVAILABLE + + +api.add_resource(HealthCheck, '/health') +# end health check + + class CompItem(Resource): - @classmethod - def get(cls): + def get(self): result = requests.get(validateuser_url + "/msapi/validateuser", cookies=request.cookies) if (result is None): - return None, 404 + return None, HTTPStatus.UNAUTHORIZED + + if (result.status_code != HTTPStatus.OK): + return result.json(), HTTPStatus.UNAUTHORIZED + + query_args_validations = { + "compitemid": fields.Int(required=True, validate=validate.Range(min=1)) + } + + parser.parse(query_args_validations, request, location="query") - if (result.status_code != 200): - return result.json(), 404 - try: - compitemid = request.args.get('compitemid',"-1") - conn = create_conn() - cursor = conn.cursor(cursor_factory = psycopg2.extras.RealDictCursor) + compitemid = request.args.get('compitemid', "-1") + conn = mypool.connect() + cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) sql = """select compid, id, name, rollup, rollback, repositoryid, target, xpos, ypos, kind, buildid, buildurl, chart, operator, builddate, dockersha, gitcommit, gitrepo, gittag, giturl, chartversion, chartnamespace, dockertag, chartrepo, @@ -54,43 +121,42 @@ def get(cls): slackchannel, discordchannel, hipchatchannel, pagerdutyurl, pagerdutybusinessurl from dm.dm_componentitem where id = %s""" - params = (compitemid,) + params = (compitemid,) cursor.execute(sql, params) result = cursor.fetchall() if (not result): - result = [OrderedDict([('compid', -1), ('id', compitemid), ('name', None), ('rollup', None), ('rollback', None), ('repositoryid', None), - ('target', None), ('xpos', None), ('ypos', None), ('kind', None), ('buildid', None), ('buildurl', None), + result = [OrderedDict([('compid', -1), ('id', compitemid), ('name', None), ('rollup', None), ('rollback', None), ('repositoryid', None), + ('target', None), ('xpos', None), ('ypos', None), ('kind', None), ('buildid', None), ('buildurl', None), ('chart', None), ('operator', None), ('builddate', None), ('dockersha', None), ('gitcommit', None), ('gitrepo', None), ('gittag', None), ('giturl', None), ('chartversion', None), ('chartnamespace', None), ('dockertag', None), ('chartrepo', None), - ('chartrepourl', None), ('serviceowner', None), ('serviceowneremail', None), ('serviceownerphone', None), + ('chartrepourl', None), ('serviceowner', None), ('serviceowneremail', None), ('serviceownerphone', None), ('slackchannel', None), ('discordchannel', None), ('hipchatchannel', None), ('pagerdutyurl', None), ('pagerdutybusinessurl', None)])] - - print(result) + cursor.close() + conn.close() return result except Exception as err: print(err) - return err + conn.rollback() + return ({"message": str(err)}), HTTPStatus.INTERNAL_SERVER_ERROR + +# Not implemented fully. SQL query is not complete + def post(self): + + result = requests.get(validateuser_url + "/msapi/validateuser", cookies=request.cookies) + if (result is None): + return None, HTTPStatus.UNAUTHORIZED + + if (result.status_code != HTTPStatus.OK): + return result.json(), HTTPStatus.UNAUTHORIZED - @classmethod - def post(cls): # completed try: - - result = requests.get(validateuser_url + "/msapi/validateuser", cookies=request.cookies) - - if (result is None): - return None, 404 - - if (result.status_code != 200): - return result.json(), 404 - - input = request.get_json(); + data = request.get_json() data_list = [] - for i in input: - d = (i['id'], i['compid'], i['status'], i['buildid'], i['buildurl'], i['dockersha'], i['dockertag'], i['gitcommit'], i['gitrepo'], i['giturl']) # this will be changed - data_list.append(d) + for col in data: + row = (col['id'], col['compid'], col['status'], col['buildid'], col['buildurl'], col['dockersha'], col['dockertag'], col['gitcommit'], col['gitrepo'], col['giturl']) # this will be changed + data_list.append(row) - print (data_list) - conn = create_conn() + conn = mypool.connect() cursor = conn.cursor() # execute the INSERT statement records_list_template = ','.join(['%s'] * len(data_list)) @@ -101,95 +167,97 @@ def post(cls): # completed rows_inserted = cursor.rowcount # Commit the changes to the database conn.commit() - return rows_inserted + conn.close() + if rows_inserted > 0: + return ({"message": 'components updated succesfully'}), HTTPStatus.CREATED + + return ({"message": 'components not updated'}), HTTPStatus.OK except Exception as err: print(err) - conn = create_conn() - cursor = conn.cursor() - cursor.execute("ROLLBACK") - conn.commit() - return err + conn.rollback() + return ({"message": str(err)}), HTTPStatus.INTERNAL_SERVER_ERROR + + def delete(self): + + result = requests.get(validateuser_url + "/msapi/validateuser", cookies=request.cookies) + if (result is None): + return None, HTTPStatus.UNAUTHORIZED + + if (result.status_code != HTTPStatus.OK): + return result.json(), HTTPStatus.UNAUTHORIZED + + query_args_validations = { + "compid": fields.Int(required=True, validate=validate.Range(min=1)) + } + + parser.parse(query_args_validations, request, location="query") - @classmethod - def delete(cls): try: - result = requests.get(validateuser_url + "/msapi/validateuser", cookies=request.cookies) - if (result is None): - return None, 404 - - if (result.status_code != 200): - return result.json(), 404 - - comp_id = request.args.get('comp_id') - #comp_item_id = request.args.get('comp_item_id') - conn = create_conn() + compid = request.args.get('compid', "-1") + conn = create_conn() cursor = conn.cursor() - sql = "select id from dm.dm_componentitem where compid = " + comp_id - t = tuple() - l = [] - cursor.execute(sql) - row = cursor.fetchone() - while row: - print (row) - l = list(t) - l.append(row[0]) - t = tuple(l) - row = cursor.fetchone() - - sql1 = "DELETE from dm.dm_compitemprops where compitemid in " + str(t) - sql2 = "DELETE from dm.dm_componentitem where compid=" + comp_id + + sql1 = "DELETE from dm.dm_compitemprops where compitemid in (select id from dm.dm_componentitem where compid = " + str(compid) + ")" + sql2 = "DELETE from dm.dm_componentitem where compid=" + compid rows_deleted = 0 - with conn.cursor() as cursor: - cursor.execute(sql1) - cursor.execute(sql2) - rows_deleted = cursor.rowcount + cursor.execute(sql1) + cursor.execute(sql2) + rows_deleted = cursor.rowcount # Commit the changes to the database conn.commit() - return rows_deleted + if rows_deleted > 0: + return ({"message": 'components updated succesfully'}), HTTPStatus.OK + + return ({"message": 'components not updated'}), HTTPStatus.OK + except Exception as err: print(err) - return err + conn.rollback() + return ({"message": str(err)}), HTTPStatus.INTERNAL_SERVER_ERROR + +# Not implemented fully. SQL query is not complete + def put(self): # not completed + + result = requests.get(validateuser_url + "/msapi/validateuser", cookies=request.cookies) + if (result is None): + return None, HTTPStatus.UNAUTHORIZED + + if (result.status_code != HTTPStatus.OK): + return result.json(), HTTPStatus.UNAUTHORIZED - @classmethod - def put(cls): # not completed try: - - result = requests.get(validateuser_url + "/msapi/validateuser", cookies=request.cookies) - if (result is None): - return None, 404 - - if (result.status_code != 200): - return result.json(), 404 - - input = request.get_json(); + conn = mypool.connect() + cursor = conn.cursor() + + data = request.get_json() data_list = [] - # for i in input: - # d = (i['id'], i['compid'], i['status'], i['buildid'], i['buildurl'], i['dockersha'], i['dockertag'], i['gitcommit'], i['gitrepo'], i['giturl']) # this will be changed - # data_list.append(d) + for col in data: + row = (col['id'], col['compid'], col['status'], col['buildid'], col['buildurl'], col['dockersha'], col['dockertag'], col['gitcommit'], col['gitrepo'], col['giturl']) # this will be changed + data_list.append(row) # print (data_list) - conn = create_conn() + conn = create_conn() cursor = conn.cursor() # # execute the INSERT statement - # records_list_template = ','.join(['%s'] * len(data_list)) - # sql = 'INSERT INTO dm.dm_componentitem(id, compid, status, buildid, buildurl, dockersha, dockertag, gitcommit, gitrepo, giturl) \ - # VALUES {}'.format(records_list_template) + records_list_template = ','.join(['%s'] * len(data_list)) + sql = 'INSERT INTO dm.dm_componentitem(id, compid, status, buildid, buildurl, dockersha, dockertag, gitcommit, gitrepo, giturl) VALUES {}'.format(records_list_template) cursor.execute(sql, data_list) # commit the changes to the database rows_inserted = cursor.rowcount # Commit the changes to the database conn.commit() - return rows_inserted + if rows_inserted > 0: + return ({"message": 'components updated succesfully'}), HTTPStatus.CREATED + + return ({"message": 'components not updated'}), HTTPStatus.OK except Exception as err: print(err) - conn = create_conn() - cursor = conn.cursor() - cursor.execute("ROLLBACK") - conn.commit() - return err - + conn.rollback() + return ({"message": str(err)}), HTTPStatus.INTERNAL_SERVER_ERROR + + ## # Actually setup the Api resource routing here ## diff --git a/requirements.txt b/requirements.txt index 8bad8efc..8b6d2749 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,14 @@ aniso8601==9.0.1 click==7.1.2 +Flask-RESTful==0.3.9 +flask-swagger-ui==3.36.0 Flask==1.1.2 -Flask-RESTful==0.3.8 itsdangerous==1.1.0 Jinja2==2.11.3 MarkupSafe==1.1.1 -psycopg2-binary==2.8.6 +# psycopg2==2.8.6 - built and installed on the fly in the docker build +pybreaker==0.6.0 pytz==2021.1 six==1.15.0 -Werkzeug==1.0.1 +sqlalchemy==1.4.22 +webargs==8.0.1 diff --git a/static/swagger-ui/doc/swagger.yml b/static/swagger-ui/doc/swagger.yml new file mode 100644 index 00000000..f889ce21 --- /dev/null +++ b/static/swagger-ui/doc/swagger.yml @@ -0,0 +1,177 @@ +openapi: 3.0.1 +info: + title: ortelius-ms-compitem-crud + version: 1.0.0 + description: This service will retrieve and delete component items and the component item properties. + termsOfService: http://swagger.io/terms/ + contact: + email: steve@deployhub.com + license: + name: Apache 2.0 + url: http://www.apache.org/licenses/LICENSE-2.0.html +externalDocs: + description: Find out more about Swagger + url: http://swagger.io +servers: +- url: / +paths: + "/msapi/compitem": + get: + tags: + - compitem + summary: Retrieves the component item and component item properties for the compitemid. + operationId: getCompItem + parameters: + - name: compitemid + in: query + required: true + schema: + type: integer + format: int32 + responses: + '200': + description: OK + content: + application/json: + schema: + type: array + items: + "$ref": "#/components/schemas/CompItem" + example: + '[{ + "compid": 2447, + "id": 5291, + "name": "authentication-service;develop;v9_0_0_62_g75d73df item 1", + "rollup": null, + "rollback": null, + "repositoryid": null, + "target": null, + "xpos": 100, + "ypos": 100, + "kind": "docker", + "buildid": "", + "buildurl": "", + "chart": null, + "operator": null, + "builddate": "Thu Aug 12 14:12:12 2021", + "dockersha": null, + "gitcommit": "", + "gitrepo": "ortelius/authentication-service", + "gittag": "", + "giturl": "", + "chartversion": null, + "chartnamespace": null, + "dockertag": null, + "chartrepo": null, + "chartrepourl": null, + "serviceowner": null, + "serviceowneremail": null, + "serviceownerphone": null, + "slackchannel": null, + "discordchannel": null, + "hipchatchannel": null, + "pagerdutyurl": null, + "pagerdutybusinessurl": null + }]' + delete: + tags: + - compitem + summary: Deletes all of the component items and component item properties for the Component (compid). + operationId: deleteCompItem + parameters: + - name: compid + in: query + required: true + schema: + type: integer + format: int32 + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + message: + type: string + example: + '{"message": "components updated succesfully"}' +components: + schemas: + CompItem: + type: object + properties: + compid: + type: integer + format: int32 + id: + type: integer + format: int32 + name: + type: string + rollup: + type: string + rollback: + type: string + repositoryid: + type: integer + format: int32 + target: + type: integer + format: int32 + xpos: + type: integer + format: int32 + ypos: + type: integer + format: int32 + kind: + type: string + buildid: + type: string + buildurl: + type: string + chart: + type: string + operator: + type: string + builddate: + type: string + dockersha: + type: string + gitcommit: + type: string + gitrepo: + type: string + gittag: + type: string + giturl: + type: string + chartversion: + type: string + chartnamespace: + type: string + dockertag: + type: string + chartrepo: + type: string + chartrepourl: + type: string + serviceowner: + type: string + serviceowneremail: + type: string + serviceownerphone: + type: string + slackchannel: + type: string + discordchannel: + type: string + hipchatchannel: + type: string + pagerdutyurl: + type: string + pagerdutybusinessurl: + type: string + \ No newline at end of file From 66f90c798b098103c438c3c8fbabcc12c0ac830c Mon Sep 17 00:00:00 2001 From: Steve Taylor Date: Fri, 27 Aug 2021 13:35:37 -0600 Subject: [PATCH 2/9] remove pycache --- __pycache__/main.cpython-38.pyc | Bin 7970 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 __pycache__/main.cpython-38.pyc diff --git a/__pycache__/main.cpython-38.pyc b/__pycache__/main.cpython-38.pyc deleted file mode 100644 index 996b2e6f4ceec9c41b386676ff98cec0bb19db32..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7970 zcmc&(TXP#ncAgssg9}OUF6w6ZB3Y1m)6MZ}Wk(cATdbu?EiaaLlFfLCZju8IFi_7x z60x9?to)KxG3DCleE{?3w>;$6OsZ0;+9$v5L!Ok{b-vRBf*?&-C8@0qP;=(=b-H`b zIp5bkn|*y*1;4LmORjlCQGQ2>-k*uYLtNp1s*1uCruvFacg%Lw!Y(wg* zpR!Z3p4qvl?X*l=wk7YGYSzxmx`v;t=Iwm7&+e=C+x;>xDS{<{;WZLq_s~7AG)d_n-Rro~ZBFnJs*UD?vzQptVGVWJ+fnUM>D$DUJ zmA9NCztVo2<&RX>$NFW>ci4daE-TpY@nQZhe~%5aAx~w)hq`?YEw1xx{9QJ3hKCQ{fz&Ut%cwIc^(*Wu^eH|qQ*W{2k%p1Ybkc9L zcVv2&-;(9;viD^AHcwaXuxso(d;d_g@AA9s1}n*t-{aZJhx{I&gtv&4yaBZ!jc}RKK0#jFHQ^otii)5 z9u9epIol$rI?NNi90g*(q{aGN-HWX?9tI6j=ICX7heuBM!rj^7qTxAPeuLMASBp^B z-1fN7!q|H5`W|y5j=Yq+6^JM{>#hj7DC;yf{4`uCTn4UrT;T{rq(o{(Ybh0-sd73B zrlZkJOfEzq!jd8Ks3)kGIQM97!C71U`R3yKMr>7i7`i(=ETv){m17oV!yF7`OY9tX=M6iNbFcF^oRn+@9IQD8rdB;?Mcaju*{f@Z9$r;Z!x8DhJiN z7xC&;Sv1&8)%9xA_5Jwru3KY1cRCB{&`iDBjw2s1gkD3MtfGlg$WNQY=-iFG^2~#} z8|~gF@ktrJ+WnDE6O_wB&K}{twv^YJT#VREnwL~Tu8y@as0%s#M9@mcIr%n|s#lP4 zJcbLWUU$Pft^^QF#Q=iX$>8yUSxc2Gt z>eE=gZRcg~#?t4DtD755RLiFYP`I4)z^+RUR~=GK#xD zYLLby-wUl{q*T;Mi}c8dQjrOhO&ut$j>)ECLr&d-8@O9;$YY}c%ZjbK8-_0e!SLF! zxTQ35!u@hk-?<&9v7j|LUYurI4z-pFy$Gqu_Ebb&$Q;=N41n7GzyQjE!vkULwc0}r zHY86|v=LQI;a<`a#@&}r@SKZnKX_qh>OtT;q1VK-&YQe!4Dk?`^i{6dDpPEYsZ9G? zv2~`yJdHT_n7e+oJHN}zd#9|L9#5++SMM7r&6P-Lt4GR#+EUvZ?pjN0!|{()RY8t% zq~NYqQcT-3#6zYtgQb3}MP?=Q>npdN2OMi{N zdh40WQ{@-RKf{w4YQ56GM5?BJi4+EJtu(Fm#pT8Mjp8jqFWc5OpESpNU4ZfWRDEw} zs@z}{?vi+7=ka2ndu1+tM$A%6V`25_N+}nc)qPr6aUGd(n1%=g5vTb}UT#D@PL=%t zmSKtDMLB5Hq9m?qt3Qi}Vf0@t%`ZBeD|27WEiKPIT3(DZi!aN(9(h46P9-7Nh>ei@ z+hP*ki`ygyNl+;3i3K_$_UFh7#~>6{Q!V_n@)}YLS^-*fpy$0!qy7rsrRB&OIZm8) zoNB-t2y#xGcbsPp*C(@d9P!icqfS3m%%ETymkd0XD$~vM4B{=!4v&40#M5&6o4ZCQml{9o~PZtkig_)yn3Dcu&CC7{?U+JTf9I7mRsCnvv7WG#K zT8IP8Vwpp-y??1p78xMrAk}sf<>}Sng0gNU{8Z+0wVwgLEs^ z9&eegG+{gR3yD{JqlRWQLCgoWG&aBr#CF&)8zEl9z-QP5yU4&<7`O@pLt$Vg3``{9 zAH+P^O=1{qYUl0O>W+#Tr`i{#XBnNlO1=c&xwMxBpJ4DF3p2PGU2a+AJuRbsh3d2E z=qlyF<5HEkwzX59WWBOZdD6_vo8g&i=+?cNVsysVT(IZyFwP!hsmC_NTX;_> z)T>_YTMh02Mgj)HL=O3J3iL$mqu8RrPuBlNo4Qp}?f%4_oNZTf71sjH!W$4ki?VYz zS@dv`p-vV>;QNhwa^G^xd(b$*$Rn0RwIWx*>L-gYkvUm}(}t7T;(5P4uZHSd4bNxP zPAY)E$zple73d(SbK#=bP7$%7$s!BNd!QA&?qm_10;8*X(U0_xavGVe7`Z$0e$qd} zb%+({JdLIn*4=XV39^A4i0tG18Fb5Kx1HD(yr^-Exi-&6pRK0dw8-u2+X1b<=f_HNhIuz8pn2wbcN30+mDV`A;E&T4dE}Rt2ljZh zB2HF(u?wn;H(va>crB#poEX7(-{Ti+9tCWJ7HkE0Zzkpdm4dJDlSNQ|zSk(I)J*Jx z$_z3MsgQM0p+UAlO1fhXq+7=pK~+ zLdOfHY*Mj8l$)22=-35i^vjH{X;Ae{@`R+Zosku2rYG$wb=+a^3#S@`y&2Ldevm8e zLiW~c1eDQ#+74vaX;YBRy6?X;h>>nak3p=&=?=>;6_Y4|eM=}IC@%B{8V9`y+Fgi8 zByjttN*L4xF_c(e0u(2)XU(m_O#3J|C3i5ffj|+8 zTgZ{1o}lhEs$lY$k#Ix1e+?K%TdsKt_u>u;!7w@5zV8b}DMj2R@ezrikaz+S>tQ6~ z(WRA*#kG|=Fj_DGFyFI6CE0g)0sG;LMRCAPRIayeCErY zms<_bV+L~Rw$LW@A&4A;7if94U)Pk8?+v4%X=*?63aXJd)jTv)9a1%*b?1MgpJ-He zoI1WM+J9Llu?}ia_hx>+1KLD3P62lwBcbb02Hed!(%V@;sc0+%oS0I#u!#OXlsR|k^Mi@&BpUZp2X*a(Y_It5`?3>)P# z54Rh>zyD^er@R(K$8}1V#V={1lJNcwX{Q5>a47F95)@B_3lUo|ZifO(Y+_H1^OZA! z15qdNYO(Hf)YuM0)s1Ah5cjAxAsFTsI z5%(b--@{H)F9$x(M1mJR;b9b~Jb>s>)^LEKbswuC_NaqCR1F=khS-krLp(Gu0!iYg1-6VNI?Ca3I!C6yL-k5 ze_js%>$4pGwPf&;$){K@8Yb5sS6F@=j?em9I__!Ke_(-OkF-p=izO|cXq8cv80|6X z-WKb}p7!5~(f2wt(u2Keu=2#YKLjh!beucWVdaT)4`3X~k*zMCaqi5k%qiy{>UD1N z>x6iV1*qav5|2qNk@y)zb7uj&;f=-O8OlX$t1g0i*r8nHFO&!B<<*KeIusnKGwse& zHQA2n+8edhoI3wm=ePOz#GOSz!>vPwN0!eEmn7);yd{=~taZAAlc(C*WK zzy*oGgRN8#0uOZ%SZ~8^fZnv-)+-}GVC@X@4*idj7iS9xr;%6tvm>vl;^m1NMDy4G zZ{T|l*!Fnm{{cSFfTegwlN2PRPllvLBpM{nIOYq=JL8?_V&w@SikH-8pG1?yuSm2= z5G0BN66c<;)f>^jq42DGN*uI+cpWU%$wf!toCyw|LctS0jLud{Y4L!b_A!Z1NJ!f8 zinPBY@%JQ#Pm2*s{w2g$xIzm8&z`idTkp%<=0N92<~Ux{`5t}4aGYiu2jcYi{sDcK zH1ReXi}xT(4`M9{;~dxn9jwSr25n$)x{bqlxw8_q`LPF%PqC*vyP!;ce+wtjIQEcX z*&aa9bA9?8vp-EoqC_7cQ`__vO3>+WLa6AYhTSJ~@kJ$!K*|K2x(k|+T}TQ#Un?9O z6VW-JoqoZ$9_t`#HiQ?aK`F67 z^|&7!Gi>J^>}|tcoL!{^J`Hejf%?g8SzSBLXx4~17)4eD8 zw4#9<*@Gv=o%4LVf8G=2h8I0b-itjVtCW*W=lE@rZ`qljH+X|Tp`&ZL)uq#1JMS_^ z??iqHlr{xM+35Z9AT5&YFt#2f2gaWOOq6gJ!omhp(m`Mp=_A-y=5ZAaAXbX Date: Fri, 27 Aug 2021 13:53:09 -0600 Subject: [PATCH 3/9] move swagger.yml to correct dir. cleanup swagger examples --- static/{swagger-ui/doc => }/swagger.yml | 74 ++++++++++++------------- 1 file changed, 35 insertions(+), 39 deletions(-) rename static/{swagger-ui/doc => }/swagger.yml (68%) diff --git a/static/swagger-ui/doc/swagger.yml b/static/swagger.yml similarity index 68% rename from static/swagger-ui/doc/swagger.yml rename to static/swagger.yml index f889ce21..38e51d87 100644 --- a/static/swagger-ui/doc/swagger.yml +++ b/static/swagger.yml @@ -37,42 +37,6 @@ paths: type: array items: "$ref": "#/components/schemas/CompItem" - example: - '[{ - "compid": 2447, - "id": 5291, - "name": "authentication-service;develop;v9_0_0_62_g75d73df item 1", - "rollup": null, - "rollback": null, - "repositoryid": null, - "target": null, - "xpos": 100, - "ypos": 100, - "kind": "docker", - "buildid": "", - "buildurl": "", - "chart": null, - "operator": null, - "builddate": "Thu Aug 12 14:12:12 2021", - "dockersha": null, - "gitcommit": "", - "gitrepo": "ortelius/authentication-service", - "gittag": "", - "giturl": "", - "chartversion": null, - "chartnamespace": null, - "dockertag": null, - "chartrepo": null, - "chartrepourl": null, - "serviceowner": null, - "serviceowneremail": null, - "serviceownerphone": null, - "slackchannel": null, - "discordchannel": null, - "hipchatchannel": null, - "pagerdutyurl": null, - "pagerdutybusinessurl": null - }]' delete: tags: - compitem @@ -95,8 +59,7 @@ paths: properties: message: type: string - example: - '{"message": "components updated succesfully"}' + example: "components updated succesfully" components: schemas: CompItem: @@ -174,4 +137,37 @@ components: type: string pagerdutybusinessurl: type: string - \ No newline at end of file + example: + compid: 2447 + id: 5291 + name: "authentication-service;develop;v9_0_0_62_g75d73df item 1" + rollup: null + rollback: null + repositoryid: null + target: null + xpos: 100 + ypos: 100 + kind: "docker" + buildid: "" + buildurl: "" + chart: null + operator: null + builddate: "Thu Aug 12 14:12:12 2021" + dockersha: null + gitcommit: "" + gitrepo: "ortelius/authentication-service" + gittag: "" + giturl: "" + chartversion: null + chartnamespace: null + dockertag: null + chartrepo: null + chartrepourl: null + serviceowner: null + serviceowneremail: null + serviceownerphone: null + slackchannel: null, + discordchannel: null + hipchatchannel: null + pagerdutyurl: null + pagerdutybusinessurl: null From f1973662ccc8a5bb9d9c77d6d05ac3f6f0f06559 Mon Sep 17 00:00:00 2001 From: Steve Taylor Date: Fri, 27 Aug 2021 15:55:06 -0600 Subject: [PATCH 4/9] use latest base image that include greenlet --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 28cd0eed..e9a1d1aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/ortelius/ms-python-base:flask-1.0 +FROM quay.io/ortelius/ms-python-base:flask-1.1 ENV DB_HOST localhost ENV DB_NAME postgres From af7de8b30a9fb0491cab63ad9222908b8e56e3d8 Mon Sep 17 00:00:00 2001 From: Utkarsh Kumar Sharma Date: Sun, 5 Sep 2021 01:17:00 +0530 Subject: [PATCH 5/9] add fastApi changes, update base image in dockerfile --- Dockerfile | 2 +- main.py | 533 +++++++++++++++++++++++++++++------------------ requirements.txt | 20 +- 3 files changed, 340 insertions(+), 215 deletions(-) diff --git a/Dockerfile b/Dockerfile index e9a1d1aa..25f5a30d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/ortelius/ms-python-base:flask-1.1 +FROM quay.io/ortelius/ms-python-base:fastapi-1.0 as base ENV DB_HOST localhost ENV DB_NAME postgres diff --git a/main.py b/main.py index 333063ee..bb9f5665 100644 --- a/main.py +++ b/main.py @@ -1,47 +1,23 @@ import os from collections import OrderedDict -from http import HTTPStatus +import uvicorn import psycopg2 import psycopg2.extras -import pybreaker import requests -import sqlalchemy.pool as pool -from flask import Flask, request, send_from_directory -from flask_restful import Api, Resource -from flask_swagger_ui import get_swaggerui_blueprint -from webargs import fields, validate -from webargs.flaskparser import abort, parser +from sqlalchemy import create_engine +from fastapi import FastAPI, Query, Request, Response, HTTPException, status +from fastapi.openapi.utils import get_openapi +from pydantic import BaseModel +from typing import List, Optional +# Init Globals +service_name = 'ortelius-ms-compitem-crud' -#pylint: disable=unused-argument -@parser.error_handler -def handle_request_parsing_error(err, req, schema, *, error_status_code, error_headers): - abort(HTTPStatus.BAD_REQUEST, errors=err.messages) - - -# Init Flask -app = Flask(__name__) -api = Api(app) -app.url_map.strict_slashes = False - - -@app.route('/static/') -def send_static(path): - return send_from_directory('static', path) - - -# swagger config -SWAGGER_URL = '/swagger' -API_URL = '/static/swagger.yml' -SWAGGERUI_BLUEPRINT = get_swaggerui_blueprint( - SWAGGER_URL, - API_URL, - config={ - 'app_name': "ortelius-ms-compitem-crud" - } -) -app.register_blueprint(SWAGGERUI_BLUEPRINT, url_prefix=SWAGGER_URL) +app = FastAPI( + title=service_name, + description=service_name + ) # Init db connection db_host = os.getenv("DB_HOST", "localhost") @@ -51,112 +27,233 @@ def send_static(path): db_port = os.getenv("DB_PORT", "5432") validateuser_url = os.getenv("VALIDATEUSER_URL", "http://localhost:5000") -# connection pool config -conn_pool_size = int(os.getenv("POOL_SIZE", "3")) -conn_pool_max_overflow = int(os.getenv("POOL_MAX_OVERFLOW", "2")) -conn_pool_timeout = float(os.getenv("POOL_TIMEOUT", "30.0")) -conn_circuit_breaker = pybreaker.CircuitBreaker( - fail_max=1, - reset_timeout=10 -) - - -@conn_circuit_breaker -def create_conn(): - conn = psycopg2.connect(host=db_host, database=db_name, user=db_user, password=db_pass, port=db_port) - return conn - +engine = create_engine("postgresql+psycopg2://" + db_user + ":" + db_pass + "@" + db_host +":"+ db_port + "/" + db_name) + +#adding custom details +def custom_openapi(): + if app.openapi_schema: + return app.openapi_schema + openapi_schema = get_openapi( + title=service_name, + version="", + description=service_name, + routes=app.routes, + ) + openapi_schema["info"]["x-logo"] = { + "url": "" + } + app.openapi_schema = openapi_schema + return app.openapi_schema -# connection pool init -mypool = pool.QueuePool(create_conn, max_overflow=conn_pool_max_overflow, pool_size=conn_pool_size, timeout=conn_pool_timeout) +app.openapi = custom_openapi # health check endpoint - - -class HealthCheck(Resource): - def get(self): - try: - conn = mypool.connect() +class StatusMsg(BaseModel): + status: str + service_name: Optional[str] = None + +@app.get("/health", + responses={ + 503: {"model": StatusMsg, + "description": "DOWN Status for the Service", + "content": { + "application/json": { + "example": {"status": 'DOWN'} + }, + }, + }, + 200: {"model": StatusMsg, + "description": "UP Status for the Service", + "content": { + "application/json": { + "example": {"status": 'UP', "service_name": service_name} + } + }, + }, + } + ) +async def health(response: Response): + try: + with engine.connect() as connection: + conn = connection.connection cursor = conn.cursor() cursor.execute('SELECT 1') - conn.close() if cursor.rowcount > 0: - return ({"status": 'UP', "service_name": 'ortelius-ms-dep-pkg-cud'}), HTTPStatus.OK - return ({"status": 'DOWN'}), HTTPStatus.SERVICE_UNAVAILABLE - - except Exception as err: - print(err) - return ({"status": 'DOWN'}), HTTPStatus.SERVICE_UNAVAILABLE - - -api.add_resource(HealthCheck, '/health') + return {"status": 'UP', "service_name": service_name} + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + return {"status": 'DOWN'} + + except Exception as err: + print(str(err)) + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + return {"status": 'DOWN'} # end health check - -class CompItem(Resource): - - def get(self): - +class CompItemModel(BaseModel): + compid: int + id: int + repositoryid: Optional[int] = None + target: Optional[str] = None + name: Optional[str] = None + summary: Optional[str] = None + predecessorid: Optional[int] = None + xpos: Optional[int] = None + ypos: Optional[int] = None + creatorid: Optional[int] = None + created: Optional[int] = None + modifierid: Optional[int] = None + modified: Optional[int] = None + status: str + rollup: Optional[int] = None + rollback: Optional[int] = None + kind: Optional[str] = None + buildid: Optional[str] = None + buildurl: Optional[str] = None + chart: Optional[str] = None + operator: Optional[str] = None + builddate: Optional[str] = None + dockersha: Optional[str] = None + gitcommit: Optional[str] = None + gitrepo: Optional[str] = None + gittag: Optional[str] = None + giturl: Optional[str] = None + dockerrepo: Optional[str] = None + chartversion: Optional[str] = None + chartnamespace: Optional[str] = None + dockertag: Optional[str] = None + chartrepo: Optional[str] = None + chartrepourl: Optional[str] = None + serviceowner: Optional[str] = None + serviceowneremail: Optional[str] = None + serviceownerphone: Optional[str] = None + slackchannel: Optional[str] = None + discordchannel: Optional[str] = None + hipchatchannel: Optional[str] = None + pagerdutyurl: Optional[str] = None + pagerdutybusinessurl: Optional[str] = None + +class CompItemModelList(BaseModel): + data: List[CompItemModel] + +class Message(BaseModel): + detail: str + + +@app.get('/msapi/compitem', + response_model=CompItemModel, + responses={ + 401: {"model": Message, + "description": "Authorization Status", + "content": { + "application/json": { + "example": {"detail": "Authorization failed"} + }, + }, + }, + 500: {"model": Message, + "description": "SQL Error", + "content": { + "application/json": { + "example": {"detail": "SQL Error: 30x"} + }, + }, + } #, + # 200: { + # "description": "List of domain ids the user belongs to.", + # "content": { + # "application/json": { + # "example": [1, 200, 201, 5033] + # } + # }, + # }, + } + ) +async def get_compitem(request: Request, compitemid:int): + try: result = requests.get(validateuser_url + "/msapi/validateuser", cookies=request.cookies) if (result is None): - return None, HTTPStatus.UNAUTHORIZED - - if (result.status_code != HTTPStatus.OK): - return result.json(), HTTPStatus.UNAUTHORIZED - - query_args_validations = { - "compitemid": fields.Int(required=True, validate=validate.Range(min=1)) - } - - parser.parse(query_args_validations, request, location="query") - - try: - compitemid = request.args.get('compitemid', "-1") - conn = mypool.connect() + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed") + + if (result.status_code != status.HTTP_200_OK): + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed status_code=" + str(result.status_code)) + except Exception as err: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed:" + str(err)) from None + + try: + with engine.connect() as connection: + conn = connection.connection + authorized = False # init to not authorized cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) sql = """select compid, id, name, rollup, rollback, repositoryid, target, xpos, ypos, - kind, buildid, buildurl, chart, operator, builddate, dockersha, gitcommit, - gitrepo, gittag, giturl, chartversion, chartnamespace, dockertag, chartrepo, - chartrepourl, serviceowner, serviceowneremail, serviceownerphone, - slackchannel, discordchannel, hipchatchannel, pagerdutyurl, pagerdutybusinessurl - from dm.dm_componentitem where id = %s""" - - params = (compitemid,) + kind, buildid, buildurl, chart, operator, builddate, dockersha, gitcommit, + gitrepo, gittag, giturl, chartversion, chartnamespace, dockertag, chartrepo, + chartrepourl, serviceowner, serviceowneremail, serviceownerphone, + slackchannel, discordchannel, hipchatchannel, pagerdutyurl, pagerdutybusinessurl + from dm.dm_componentitem where id = %s""" + + params = (str(compitemid),) cursor.execute(sql, params) result = cursor.fetchall() if (not result): result = [OrderedDict([('compid', -1), ('id', compitemid), ('name', None), ('rollup', None), ('rollback', None), ('repositoryid', None), - ('target', None), ('xpos', None), ('ypos', None), ('kind', None), ('buildid', None), ('buildurl', None), - ('chart', None), ('operator', None), ('builddate', None), ('dockersha', None), ('gitcommit', None), - ('gitrepo', None), ('gittag', None), ('giturl', None), ('chartversion', None), ('chartnamespace', None), ('dockertag', None), ('chartrepo', None), - ('chartrepourl', None), ('serviceowner', None), ('serviceowneremail', None), ('serviceownerphone', None), - ('slackchannel', None), ('discordchannel', None), ('hipchatchannel', None), ('pagerdutyurl', None), ('pagerdutybusinessurl', None)])] + ('target', None), ('xpos', None), ('ypos', None), ('kind', None), ('buildid', None), ('buildurl', None), + ('chart', None), ('operator', None), ('builddate', None), ('dockersha', None), ('gitcommit', None), + ('gitrepo', None), ('gittag', None), ('giturl', None), ('chartversion', None), ('chartnamespace', None), ('dockertag', None), ('chartrepo', None), + ('chartrepourl', None), ('serviceowner', None), ('serviceowneremail', None), ('serviceownerphone', None), + ('slackchannel', None), ('discordchannel', None), ('hipchatchannel', None), ('pagerdutyurl', None), ('pagerdutybusinessurl', None)])] cursor.close() conn.close() - return result - except Exception as err: - print(err) - conn.rollback() - return ({"message": str(err)}), HTTPStatus.INTERNAL_SERVER_ERROR + return result[0] + + except HTTPException: + raise + except Exception as err: + print(err) + # conn.rollback() + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(err)) from None # Not implemented fully. SQL query is not complete - def post(self): +@app.post("/msapi/compitem", + responses={ + 401: {"model": Message, + "description": "Authorization Status", + "content": { + "application/json": { + "example": {"detail": "Authorization failed"} + }, + }, + }, + 500: {"model": Message, + "description": "SQL Error", + "content": { + "application/json": { + "example": {"detail": "SQL Error: 30x"} + }, + }, + } + } + ) +async def create_compitem(response: Response, request: Request, compItemList: List[CompItemModel]): + + try: result = requests.get(validateuser_url + "/msapi/validateuser", cookies=request.cookies) if (result is None): - return None, HTTPStatus.UNAUTHORIZED - - if (result.status_code != HTTPStatus.OK): - return result.json(), HTTPStatus.UNAUTHORIZED - - try: - data = request.get_json() - data_list = [] - for col in data: - row = (col['id'], col['compid'], col['status'], col['buildid'], col['buildurl'], col['dockersha'], col['dockertag'], col['gitcommit'], col['gitrepo'], col['giturl']) # this will be changed - data_list.append(row) - - conn = mypool.connect() + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed") + + if (result.status_code != status.HTTP_200_OK): + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed status_code=" + str(result.status_code)) + except Exception as err: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed:" + str(err)) from None + + try: + data_list = [] + for col in compItemList: + row = (col.id, col.compid, col.status, col.buildid, col.buildurl, col.dockersha, col.dockertag, col.gitcommit, col.gitrepo, col.giturl) # this will be changed + data_list.append(row) + + with engine.connect() as connection: + conn = connection.connection cursor = conn.cursor() # execute the INSERT statement records_list_template = ','.join(['%s'] * len(data_list)) @@ -168,100 +265,136 @@ def post(self): # Commit the changes to the database conn.commit() conn.close() - if rows_inserted > 0: - return ({"message": 'components updated succesfully'}), HTTPStatus.CREATED - - return ({"message": 'components not updated'}), HTTPStatus.OK - - except Exception as err: - print(err) - conn.rollback() - return ({"message": str(err)}), HTTPStatus.INTERNAL_SERVER_ERROR - - def delete(self): - + + if rows_inserted > 0: + response.status_code = status.HTTP_201_CREATED + return {"message": 'components created succesfully'} + + response.status_code = status.HTTP_200_OK + return {"message": 'components not created'} + + except HTTPException: + raise + except Exception as err: + print(err) + # conn.rollback() + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(err)) from None + +@app.delete("/msapi/compitem", + responses={ + 401: {"model": Message, + "description": "Authorization Status", + "content": { + "application/json": { + "example": {"detail": "Authorization failed"} + }, + }, + }, + 500: {"model": Message, + "description": "SQL Error", + "content": { + "application/json": { + "example": {"detail": "SQL Error: 30x"} + }, + }, + } + } + ) +async def delete_compitem(compid: int): + + try: result = requests.get(validateuser_url + "/msapi/validateuser", cookies=request.cookies) if (result is None): - return None, HTTPStatus.UNAUTHORIZED - - if (result.status_code != HTTPStatus.OK): - return result.json(), HTTPStatus.UNAUTHORIZED - - query_args_validations = { - "compid": fields.Int(required=True, validate=validate.Range(min=1)) - } - - parser.parse(query_args_validations, request, location="query") - - try: - compid = request.args.get('compid', "-1") - conn = create_conn() + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed") + + if (result.status_code != status.HTTP_200_OK): + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed status_code=" + str(result.status_code)) + except Exception as err: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed:" + str(err)) from None + + try: + with engine.connect() as connection: + conn = connection.connection cursor = conn.cursor() - + sql1 = "DELETE from dm.dm_compitemprops where compitemid in (select id from dm.dm_componentitem where compid = " + str(compid) + ")" - sql2 = "DELETE from dm.dm_componentitem where compid=" + compid + sql2 = "DELETE from dm.dm_componentitem where compid=" + str(compid) rows_deleted = 0 cursor.execute(sql1) cursor.execute(sql2) rows_deleted = cursor.rowcount # Commit the changes to the database conn.commit() - if rows_deleted > 0: - return ({"message": 'components updated succesfully'}), HTTPStatus.OK - - return ({"message": 'components not updated'}), HTTPStatus.OK - - except Exception as err: - print(err) - conn.rollback() - return ({"message": str(err)}), HTTPStatus.INTERNAL_SERVER_ERROR - -# Not implemented fully. SQL query is not complete - def put(self): # not completed - - result = requests.get(validateuser_url + "/msapi/validateuser", cookies=request.cookies) - if (result is None): - return None, HTTPStatus.UNAUTHORIZED - - if (result.status_code != HTTPStatus.OK): - return result.json(), HTTPStatus.UNAUTHORIZED - - try: - conn = mypool.connect() - cursor = conn.cursor() - - data = request.get_json() - data_list = [] - for col in data: - row = (col['id'], col['compid'], col['status'], col['buildid'], col['buildurl'], col['dockersha'], col['dockertag'], col['gitcommit'], col['gitrepo'], col['giturl']) # this will be changed - data_list.append(row) - - # print (data_list) - conn = create_conn() + + # response.status_code = status.HTTP_200_OK + return {"message": 'component deleted succesfully'} + + except HTTPException: + raise + except Exception as err: + print(err) + # conn.rollback() + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(err)) from None + +@app.put("/msapi/compitem", + responses={ + 401: {"model": Message, + "description": "Authorization Status", + "content": { + "application/json": { + "example": {"detail": "Authorization failed"} + }, + }, + }, + 500: {"model": Message, + "description": "SQL Error", + "content": { + "application/json": { + "example": {"detail": "SQL Error: 30x"} + }, + }, + } + } + ) +async def update_compitem(compitemList: List[CompItemModel]): + + # try: + # result = requests.get(validateuser_url + "/msapi/validateuser", cookies=request.cookies) + # if (result is None): + # raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed") + # + # if (result.status_code != status.HTTP_200_OK): + # raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed status_code=" + str(result.status_code)) + # except Exception as err: + # raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed:" + str(err)) from None + + try: + data_list = [] + for col in compitemList: + row = (col.compid, col.status, col.buildid, col.buildurl, col.dockersha, col.dockertag, col.gitcommit, col.gitrepo, col.giturl, col.id) # this will be changed + data_list.append(row) + + with engine.connect() as connection: + conn = connection.connection cursor = conn.cursor() # # execute the INSERT statement - records_list_template = ','.join(['%s'] * len(data_list)) - sql = 'INSERT INTO dm.dm_componentitem(id, compid, status, buildid, buildurl, dockersha, dockertag, gitcommit, gitrepo, giturl) VALUES {}'.format(records_list_template) - cursor.execute(sql, data_list) + # records_list_template = ','.join(['%s'] * len(data_list)) + sql = 'UPDATE dm.dm_componentitem set compid=%s, status=%s, buildid=%s, buildurl=%s, dockersha=%s, dockertag=%s, gitcommit=%s, gitrepo=%s, giturl=%s \ + WHERE id = %s' + cursor.executemany(sql, data_list) # commit the changes to the database rows_inserted = cursor.rowcount # Commit the changes to the database conn.commit() - if rows_inserted > 0: - return ({"message": 'components updated succesfully'}), HTTPStatus.CREATED - - return ({"message": 'components not updated'}), HTTPStatus.OK - - except Exception as err: - print(err) - conn.rollback() - return ({"message": str(err)}), HTTPStatus.INTERNAL_SERVER_ERROR - - -## -# Actually setup the Api resource routing here -## -api.add_resource(CompItem, '/msapi/compitem') - -if __name__ == '__main__': - app.run(host="0.0.0.0", port=5001) + + if rows_inserted > 0: + return {"message": 'components updated succesfully'} + + return {"message": 'components not updated'} + + except Exception as err: + print(err) + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(err)) from None + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/requirements.txt b/requirements.txt index 8b6d2749..07002d48 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,6 @@ -aniso8601==9.0.1 -click==7.1.2 -Flask-RESTful==0.3.9 -flask-swagger-ui==3.36.0 -Flask==1.1.2 -itsdangerous==1.1.0 -Jinja2==2.11.3 -MarkupSafe==1.1.1 -# psycopg2==2.8.6 - built and installed on the fly in the docker build -pybreaker==0.6.0 -pytz==2021.1 -six==1.15.0 -sqlalchemy==1.4.22 -webargs==8.0.1 +# psycopg2==2.8.6 - installed by base image +#sqlalchemy - installed by base image +#fastapi - installed by base image +uvicorn==0.15.0 +pydantic==1.8.2 +requests==2.25.0 \ No newline at end of file From c9003dfb2fd381b48b16c74679661c54a0aa7437 Mon Sep 17 00:00:00 2001 From: Utkarsh Kumar Sharma Date: Tue, 7 Sep 2021 10:52:51 +0530 Subject: [PATCH 6/9] code refactoring and fixes --- main.py | 49 +++++++++++++++---------------------------------- 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/main.py b/main.py index bb9f5665..ed567115 100644 --- a/main.py +++ b/main.py @@ -6,8 +6,7 @@ import psycopg2.extras import requests from sqlalchemy import create_engine -from fastapi import FastAPI, Query, Request, Response, HTTPException, status -from fastapi.openapi.utils import get_openapi +from fastapi import FastAPI, Request, Response, HTTPException, status from pydantic import BaseModel from typing import List, Optional @@ -29,24 +28,6 @@ engine = create_engine("postgresql+psycopg2://" + db_user + ":" + db_pass + "@" + db_host +":"+ db_port + "/" + db_name) -#adding custom details -def custom_openapi(): - if app.openapi_schema: - return app.openapi_schema - openapi_schema = get_openapi( - title=service_name, - version="", - description=service_name, - routes=app.routes, - ) - openapi_schema["info"]["x-logo"] = { - "url": "" - } - app.openapi_schema = openapi_schema - return app.openapi_schema - -app.openapi = custom_openapi - # health check endpoint class StatusMsg(BaseModel): status: str @@ -103,7 +84,7 @@ class CompItemModel(BaseModel): created: Optional[int] = None modifierid: Optional[int] = None modified: Optional[int] = None - status: str + status: Optional[str] = None rollup: Optional[int] = None rollback: Optional[int] = None kind: Optional[str] = None @@ -140,7 +121,7 @@ class Message(BaseModel): @app.get('/msapi/compitem', - response_model=CompItemModel, + response_model=List[CompItemModel], responses={ 401: {"model": Message, "description": "Authorization Status", @@ -203,7 +184,7 @@ async def get_compitem(request: Request, compitemid:int): ('slackchannel', None), ('discordchannel', None), ('hipchatchannel', None), ('pagerdutyurl', None), ('pagerdutybusinessurl', None)])] cursor.close() conn.close() - return result[0] + return result except HTTPException: raise @@ -300,7 +281,7 @@ async def create_compitem(response: Response, request: Request, compItemList: Li } } ) -async def delete_compitem(compid: int): +async def delete_compitem(request: Request, compid: int): try: result = requests.get(validateuser_url + "/msapi/validateuser", cookies=request.cookies) @@ -356,17 +337,17 @@ async def delete_compitem(compid: int): } } ) -async def update_compitem(compitemList: List[CompItemModel]): +async def update_compitem(request: Request, compitemList: List[CompItemModel]): - # try: - # result = requests.get(validateuser_url + "/msapi/validateuser", cookies=request.cookies) - # if (result is None): - # raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed") - # - # if (result.status_code != status.HTTP_200_OK): - # raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed status_code=" + str(result.status_code)) - # except Exception as err: - # raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed:" + str(err)) from None + try: + result = requests.get(validateuser_url + "/msapi/validateuser", cookies=request.cookies) + if (result is None): + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed") + + if (result.status_code != status.HTTP_200_OK): + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed status_code=" + str(result.status_code)) + except Exception as err: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed:" + str(err)) from None try: data_list = [] From cc89f6ef3f1babf9b3d36e97a3fc7777c8f9514f Mon Sep 17 00:00:00 2001 From: Steve Taylor Date: Fri, 10 Sep 2021 15:21:26 -0600 Subject: [PATCH 7/9] add chart and cloudbuild --- chart/ms-validate-user/.helmignore | 23 +++ chart/ms-validate-user/Chart.yaml | 6 + chart/ms-validate-user/templates/_helpers.tpl | 32 ++++ .../templates/deployment.yaml | 52 ++++++ chart/ms-validate-user/templates/service.yaml | 13 ++ chart/ms-validate-user/values.yaml | 31 ++++ cloudbuild/cloudbuild.yaml | 62 +++++-- static/swagger.yml | 173 ------------------ workspace.code-workspace | 8 + 9 files changed, 213 insertions(+), 187 deletions(-) create mode 100644 chart/ms-validate-user/.helmignore create mode 100644 chart/ms-validate-user/Chart.yaml create mode 100644 chart/ms-validate-user/templates/_helpers.tpl create mode 100644 chart/ms-validate-user/templates/deployment.yaml create mode 100644 chart/ms-validate-user/templates/service.yaml create mode 100644 chart/ms-validate-user/values.yaml delete mode 100644 static/swagger.yml create mode 100644 workspace.code-workspace diff --git a/chart/ms-validate-user/.helmignore b/chart/ms-validate-user/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/chart/ms-validate-user/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/chart/ms-validate-user/Chart.yaml b/chart/ms-validate-user/Chart.yaml new file mode 100644 index 00000000..79419600 --- /dev/null +++ b/chart/ms-validate-user/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: ms-compitem-crud +description: Dependency Packages +type: application +version: 0.1.0 +appVersion: "1.0" diff --git a/chart/ms-validate-user/templates/_helpers.tpl b/chart/ms-validate-user/templates/_helpers.tpl new file mode 100644 index 00000000..2c9e5706 --- /dev/null +++ b/chart/ms-validate-user/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "ms-compitem-crud.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ms-compitem-crud.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ms-compitem-crud.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + diff --git a/chart/ms-validate-user/templates/deployment.yaml b/chart/ms-validate-user/templates/deployment.yaml new file mode 100644 index 00000000..d6edc710 --- /dev/null +++ b/chart/ms-validate-user/templates/deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "ms-compitem-crud.name" . }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "ms-compitem-crud.name" . }} + tier: backend + track: stable + template: + metadata: + labels: + app: {{ include "ms-compitem-crud.name" . }} + tier: backend + track: stable + spec: + containers: + - name: {{ include "ms-compitem-crud.name" . }} + image: "{{ .Values.DockerRepo }}:{{ .Values.DockerTag }}" + imagePullPolicy: Always + env: + - name: DB_USER + valueFrom: + secretKeyRef: + name: pgcred + key: DBUserName + - name: DB_PASS + valueFrom: + secretKeyRef: + name: pgcred + key: DBPassword + - name: DB_HOST + valueFrom: + secretKeyRef: + name: pgcred + key: DBHost + - name: DB_PORT + valueFrom: + secretKeyRef: + name: pgcred + key: DBPort + - name: DB_NAME + valueFrom: + secretKeyRef: + name: pgcred + key: DBName + ports: + - name: http + containerPort: 80 +--- diff --git a/chart/ms-validate-user/templates/service.yaml b/chart/ms-validate-user/templates/service.yaml new file mode 100644 index 00000000..9b45f57c --- /dev/null +++ b/chart/ms-validate-user/templates/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "ms-compitem-crud.name" . }} +spec: + selector: + app: {{ include "ms-compitem-crud.name" . }} + ports: + - protocol: TCP + port: 80 + targetPort: 80 + type: NodePort +--- diff --git a/chart/ms-validate-user/values.yaml b/chart/ms-validate-user/values.yaml new file mode 100644 index 00000000..57a35f4f --- /dev/null +++ b/chart/ms-validate-user/values.yaml @@ -0,0 +1,31 @@ +nameOverride: "" +fullnameOverride: "" + +replicaCount: 1 + +image: + # TODO - update image params when image is pushed to repo. + repository: quay.io/ortelius/ms-compitem-crud + #digest: + tag: main-v9.0.0.17-g106d15a + +service: + type: NodePort + portName: env2app-port + exposedPort: 8080 + targetPort: 80 + nodePort: 30080 + +envVars: + DB_HOST: 192.168.10.96 + DB_PORT: 6543 + DB_NAME: postgres + + +resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi diff --git a/cloudbuild/cloudbuild.yaml b/cloudbuild/cloudbuild.yaml index 5a796152..073cc789 100644 --- a/cloudbuild/cloudbuild.yaml +++ b/cloudbuild/cloudbuild.yaml @@ -1,44 +1,73 @@ steps: # Get ssh key from Google Secret Manager - name: gcr.io/cloud-builders/gcloud - id: sshkey + id: ssh_keys entrypoint: 'bash' - args: [ '-c', 'gcloud secrets versions access latest --secret=ortelius-github > /root/.ssh/id_rsa;chmod 600 /root/.ssh/id_rsa;ssh-keyscan -t rsa github.com > /root/.ssh/known_hosts' ] + args: [ '-c', 'gcloud secrets versions access latest --secret=github > /root/.ssh/id_rsa;chmod 600 /root/.ssh/id_rsa;ssh-keyscan -t rsa github.com > /root/.ssh/known_hosts' ] volumes: - name: 'ssh' path: /root/.ssh + + # Login to Quay for push. + - name: 'gcr.io/cloud-builders/docker' + id: login + waitFor: ['ssh_keys'] + entrypoint: 'bash' + args: ['-c', 'docker login quay.io --username "$$QUAY_USERID" --password $$QUAY_PASSWORD'] + secretEnv: ['QUAY_USERID', 'QUAY_PASSWORD'] + env: + - 'DOCKER_CONFIG=/workspace/docker-config' + # Setup environment including img tag name for nginx - name: gcr.io/cloud-builders/docker - id: cloudbuild_sh + id: env + waitFor: ['login'] entrypoint: 'bash' - args: ['-c', 'ls -A1 | xargs rm -rf;git clone $$COMPONENT_GITURL .;git checkout --track -b $BRANCH_NAME origin/$BRANCH_NAME;env | sed "s/^/export /" >> /workspace/cloudbuild.sh'] + args: ['-c', 'ls -A1 | grep -v docker-config | xargs rm -rf;git init; git remote add origin $$COMPONENT_GITURL;git fetch;git checkout --track -b $BRANCH_NAME origin/$BRANCH_NAME;env | sed "s/^/export /" >> /workspace/cloudbuild.sh'] volumes: - name: 'ssh' path: /root/.ssh env: + - 'COMPONENT_APPLICATION=GLOBAL.ortelius.saas.ortelius-devops' + - 'COMPONENT_NAME=GLOBAL.ortelius.saas.ms-compitem-crud' - 'COMPONENT_GITURL=git@github.com:ortelius/ortelius-ms-compitem-crud.git' - - 'COMPONENT_VERSION=9.0.0' + - 'COMPONENT_VARIANT=$BRANCH_NAME' + - 'COMPONENT_VERSION=10.0.0' + - 'COMPONENT_VERSION_COMMIT="v$$COMPONENT_VERSION.$$(git rev-list --count $BRANCH_NAME)-g$SHORT_SHA"' - 'COMPONENT_DOCKERREPO=quay.io/ortelius/ms-compitem-crud' + - 'COMPONENT_CUSTOMACTION=GLOBAL.HelmChart' + - 'COMPONENT_CHARTNAME=chart/ms-compitem-crud' + - 'COMPONENT_CHARTNAMESPACE=ortelius' + - 'DEPLOY_ENV=GLOBAL.ortelius.saas.aks-cluster' - 'BLDDATE=`date`' - 'IMAGE_TAG="$BRANCH_NAME-v$$COMPONENT_VERSION.$$(git rev-list --count $BRANCH_NAME)-g$SHORT_SHA"' + - 'DOCKER_CONFIG=/workspace/docker-config' - # Login to Quay for push. + # Build and push quay.io/ortelius/ms-compitem-crud - name: 'gcr.io/cloud-builders/docker' - id: login + id: build_push + waitFor: ['env'] entrypoint: 'bash' - args: ['-c', '. /workspace/cloudbuild.sh;docker login quay.io --username "$$QUAY_USERID" --password $$QUAY_PASSWORD'] - secretEnv: ['QUAY_USERID', 'QUAY_PASSWORD'] + args: ["-c", '. /workspace/cloudbuild.sh;docker build --tag $$COMPONENT_DOCKERREPO:$$IMAGE_TAG -f /workspace/Dockerfile .;docker push $$COMPONENT_DOCKERREPO:$$IMAGE_TAG'] env: - 'DOCKER_CONFIG=/workspace/docker-config' - # Build and push deployhub-webadmin.war - quay.io/ortelius/ms-textfile-crud + # Get image id - name: 'gcr.io/cloud-builders/docker' - id: ortelius - waitFor: [ 'login' ] + id: digest + waitFor: ['build_push'] entrypoint: 'bash' - args: ["-c", '. /workspace/cloudbuild.sh;docker build --tag $$COMPONENT_DOCKERREPO:$$IMAGE_TAG -f /workspace/Dockerfile .;docker push $$COMPONENT_DOCKERREPO:$$IMAGE_TAG'] env: - 'DOCKER_CONFIG=/workspace/docker-config' + args: ['-c', ". /workspace/cloudbuild.sh;echo export DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' $$COMPONENT_DOCKERREPO:$$IMAGE_TAG) >> /workspace/cloudbuild.sh" ] + + # Capture new component version in DeployHub + - name: 'quay.io/deployhub/compupdate' + id: compupdate + waitFor: ['digest'] + entrypoint: 'bash' + secretEnv: ['DHUSER', 'DHPASS'] + args: ['-c', '. /workspace/cloudbuild.sh;dh updatecomp --dhurl https://console.deployhub.com --appname "$$COMPONENT_APPLICATION" --compname "$$COMPONENT_NAME" --compvariant "$$COMPONENT_VARIANT" --compversion "$$COMPONENT_VERSION_COMMIT" --deployenv "$$DEPLOY_ENV" --docker --compattr "GitCommit:$SHORT_SHA" --compattr "GitUrl:$$COMPONENT_GITURL" --compattr "GitRepo:ortelius/$REPO_NAME" --compattr "GitTag:$TAG_NAME" --compattr "GitBranch:$BRANCH_NAME" --compattr "Chart:$$COMPONENT_CHARTNAME" --compattr "DockerSha:$$DIGEST" --compattr "DockerBuildDate:$$BLDDATE" --compattr "DockerRepo:$$COMPONENT_DOCKERREPO" --compattr "BuildId:$BUILD_ID" --compattr "BuildUrl:https://console.cloud.google.com/cloud-build/builds/$BUILD_ID?project=$PROJECT_ID" --compattr "CustomAction:$$COMPONENT_CUSTOMACTION" --compattr "DockerTag:$$IMAGE_TAG" --compattr "ChartNamespace:$$COMPONENT_CHARTNAMESPACE"'] secrets: - kmsKeyName: projects/eighth-physics-169321/locations/global/keyRings/cli/cryptoKeys/quay @@ -47,4 +76,9 @@ secrets: - kmsKeyName: projects/eighth-physics-169321/locations/global/keyRings/cli/cryptoKeys/quay-pw secretEnv: QUAY_PASSWORD: CiQAUULEud9Ej8XtwNAb9gkbDVhSGFZYhUGE30fNwR+7ehAOkH8SMgCz6KYeykjgS16RPxgKlrIQL/1TKDt06v4OXGIisFXOkdWC+jvdda8mTzVNCi8sT5g6 - +- kmsKeyName: projects/eighth-physics-169321/locations/global/keyRings/cli/cryptoKeys/ortelius-id + secretEnv: + DHUSER: CiQAGgJuQMHWANazqTOeE/SyoX/YNVWnES7eJEVWY8mTP98Er3USMQC43iiopoGYhP/YahsQu/yUURiqJBVZURYiUiu5Z7UBkrDgUAonKCKjtzeSNUP7HoQ= +- kmsKeyName: projects/eighth-physics-169321/locations/global/keyRings/cli/cryptoKeys/ortelius-pw + secretEnv: + DHPASS: CiQAZySXz07McN9e6fyr6X4qwkw4iBgeULmpq16RbxIAcqg6gTESMQB98+y30zqMVPx2S/Q/8ld+qlJWWxmocnbjLe9iyepMwyMl3yf+r5e55nf85PlrBBw= diff --git a/static/swagger.yml b/static/swagger.yml deleted file mode 100644 index 38e51d87..00000000 --- a/static/swagger.yml +++ /dev/null @@ -1,173 +0,0 @@ -openapi: 3.0.1 -info: - title: ortelius-ms-compitem-crud - version: 1.0.0 - description: This service will retrieve and delete component items and the component item properties. - termsOfService: http://swagger.io/terms/ - contact: - email: steve@deployhub.com - license: - name: Apache 2.0 - url: http://www.apache.org/licenses/LICENSE-2.0.html -externalDocs: - description: Find out more about Swagger - url: http://swagger.io -servers: -- url: / -paths: - "/msapi/compitem": - get: - tags: - - compitem - summary: Retrieves the component item and component item properties for the compitemid. - operationId: getCompItem - parameters: - - name: compitemid - in: query - required: true - schema: - type: integer - format: int32 - responses: - '200': - description: OK - content: - application/json: - schema: - type: array - items: - "$ref": "#/components/schemas/CompItem" - delete: - tags: - - compitem - summary: Deletes all of the component items and component item properties for the Component (compid). - operationId: deleteCompItem - parameters: - - name: compid - in: query - required: true - schema: - type: integer - format: int32 - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - properties: - message: - type: string - example: "components updated succesfully" -components: - schemas: - CompItem: - type: object - properties: - compid: - type: integer - format: int32 - id: - type: integer - format: int32 - name: - type: string - rollup: - type: string - rollback: - type: string - repositoryid: - type: integer - format: int32 - target: - type: integer - format: int32 - xpos: - type: integer - format: int32 - ypos: - type: integer - format: int32 - kind: - type: string - buildid: - type: string - buildurl: - type: string - chart: - type: string - operator: - type: string - builddate: - type: string - dockersha: - type: string - gitcommit: - type: string - gitrepo: - type: string - gittag: - type: string - giturl: - type: string - chartversion: - type: string - chartnamespace: - type: string - dockertag: - type: string - chartrepo: - type: string - chartrepourl: - type: string - serviceowner: - type: string - serviceowneremail: - type: string - serviceownerphone: - type: string - slackchannel: - type: string - discordchannel: - type: string - hipchatchannel: - type: string - pagerdutyurl: - type: string - pagerdutybusinessurl: - type: string - example: - compid: 2447 - id: 5291 - name: "authentication-service;develop;v9_0_0_62_g75d73df item 1" - rollup: null - rollback: null - repositoryid: null - target: null - xpos: 100 - ypos: 100 - kind: "docker" - buildid: "" - buildurl: "" - chart: null - operator: null - builddate: "Thu Aug 12 14:12:12 2021" - dockersha: null - gitcommit: "" - gitrepo: "ortelius/authentication-service" - gittag: "" - giturl: "" - chartversion: null - chartnamespace: null - dockertag: null - chartrepo: null - chartrepourl: null - serviceowner: null - serviceowneremail: null - serviceownerphone: null - slackchannel: null, - discordchannel: null - hipchatchannel: null - pagerdutyurl: null - pagerdutybusinessurl: null diff --git a/workspace.code-workspace b/workspace.code-workspace new file mode 100644 index 00000000..876a1499 --- /dev/null +++ b/workspace.code-workspace @@ -0,0 +1,8 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": {} +} \ No newline at end of file From 149ac2cda67efee7a7a23325448ead2816cc6eae Mon Sep 17 00:00:00 2001 From: Utkarsh Kumar Sharma Date: Fri, 8 Oct 2021 18:13:20 +0530 Subject: [PATCH 8/9] implement connection retry logic in case of failure --- main.py | 246 ++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 166 insertions(+), 80 deletions(-) diff --git a/main.py b/main.py index ed567115..48a01976 100644 --- a/main.py +++ b/main.py @@ -9,9 +9,13 @@ from fastapi import FastAPI, Request, Response, HTTPException, status from pydantic import BaseModel from typing import List, Optional +from sqlalchemy.exc import OperationalError, StatementError +from time import sleep +import logging # Init Globals service_name = 'ortelius-ms-compitem-crud' +db_conn_retry = 3 app = FastAPI( title=service_name, @@ -26,7 +30,7 @@ db_port = os.getenv("DB_PORT", "5432") validateuser_url = os.getenv("VALIDATEUSER_URL", "http://localhost:5000") -engine = create_engine("postgresql+psycopg2://" + db_user + ":" + db_pass + "@" + db_host +":"+ db_port + "/" + db_name) +engine = create_engine("postgresql+psycopg2://" + db_user + ":" + db_pass + "@" + db_host +":"+ db_port + "/" + db_name, pool_pre_ping=True) # health check endpoint class StatusMsg(BaseModel): @@ -161,31 +165,51 @@ async def get_compitem(request: Request, compitemid:int): raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed:" + str(err)) from None try: - with engine.connect() as connection: - conn = connection.connection - authorized = False # init to not authorized - cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) - sql = """select compid, id, name, rollup, rollback, repositoryid, target, xpos, ypos, - kind, buildid, buildurl, chart, operator, builddate, dockersha, gitcommit, - gitrepo, gittag, giturl, chartversion, chartnamespace, dockertag, chartrepo, - chartrepourl, serviceowner, serviceowneremail, serviceownerphone, - slackchannel, discordchannel, hipchatchannel, pagerdutyurl, pagerdutybusinessurl - from dm.dm_componentitem where id = %s""" - - params = (str(compitemid),) - cursor.execute(sql, params) - result = cursor.fetchall() - if (not result): - result = [OrderedDict([('compid', -1), ('id', compitemid), ('name', None), ('rollup', None), ('rollback', None), ('repositoryid', None), - ('target', None), ('xpos', None), ('ypos', None), ('kind', None), ('buildid', None), ('buildurl', None), - ('chart', None), ('operator', None), ('builddate', None), ('dockersha', None), ('gitcommit', None), - ('gitrepo', None), ('gittag', None), ('giturl', None), ('chartversion', None), ('chartnamespace', None), ('dockertag', None), ('chartrepo', None), - ('chartrepourl', None), ('serviceowner', None), ('serviceowneremail', None), ('serviceownerphone', None), - ('slackchannel', None), ('discordchannel', None), ('hipchatchannel', None), ('pagerdutyurl', None), ('pagerdutybusinessurl', None)])] - cursor.close() - conn.close() - return result - + #Retry logic for failed query + no_of_retry = db_conn_retry + attempt = 1; + while True: + try: + with engine.connect() as connection: + conn = connection.connection + authorized = False # init to not authorized + cursor = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + sql = """select compid, id, name, rollup, rollback, repositoryid, target, xpos, ypos, + kind, buildid, buildurl, chart, operator, builddate, dockersha, gitcommit, + gitrepo, gittag, giturl, chartversion, chartnamespace, dockertag, chartrepo, + chartrepourl, serviceowner, serviceowneremail, serviceownerphone, + slackchannel, discordchannel, hipchatchannel, pagerdutyurl, pagerdutybusinessurl + from dm.dm_componentitem where id = %s""" + + params = (str(compitemid),) + cursor.execute(sql, params) + result = cursor.fetchall() + if (not result): + result = [OrderedDict([('compid', -1), ('id', compitemid), ('name', None), ('rollup', None), ('rollback', None), ('repositoryid', None), + ('target', None), ('xpos', None), ('ypos', None), ('kind', None), ('buildid', None), ('buildurl', None), + ('chart', None), ('operator', None), ('builddate', None), ('dockersha', None), ('gitcommit', None), + ('gitrepo', None), ('gittag', None), ('giturl', None), ('chartversion', None), ('chartnamespace', None), ('dockertag', None), ('chartrepo', None), + ('chartrepourl', None), ('serviceowner', None), ('serviceowneremail', None), ('serviceownerphone', None), + ('slackchannel', None), ('discordchannel', None), ('hipchatchannel', None), ('pagerdutyurl', None), ('pagerdutybusinessurl', None)])] + cursor.close() + conn.close() + return result + + except (InterfaceError, OperationalError) as ex: + if attempt < no_of_retry: + logging.error( + "Database connection error: {} - sleeping for {}s" + " and will retry (attempt #{} of {})".format( + ex, sleep_for, attempt, no_of_retry + ) + ) + #200ms of sleep time in cons. retry calls + sleep(0.2) + attempt += 1 + continue + else: + raise + except HTTPException: raise except Exception as err: @@ -233,27 +257,47 @@ async def create_compitem(response: Response, request: Request, compItemList: Li row = (col.id, col.compid, col.status, col.buildid, col.buildurl, col.dockersha, col.dockertag, col.gitcommit, col.gitrepo, col.giturl) # this will be changed data_list.append(row) - with engine.connect() as connection: - conn = connection.connection - cursor = conn.cursor() - # execute the INSERT statement - records_list_template = ','.join(['%s'] * len(data_list)) - sql = 'INSERT INTO dm.dm_componentitem(id, compid, status, buildid, buildurl, dockersha, dockertag, gitcommit, gitrepo, giturl) \ - VALUES {}'.format(records_list_template) - cursor.execute(sql, data_list) - # commit the changes to the database - rows_inserted = cursor.rowcount - # Commit the changes to the database - conn.commit() - conn.close() - - if rows_inserted > 0: - response.status_code = status.HTTP_201_CREATED - return {"message": 'components created succesfully'} - - response.status_code = status.HTTP_200_OK - return {"message": 'components not created'} - + records_list_template = ','.join(['%s'] * len(data_list)) + sql = 'INSERT INTO dm.dm_componentitem(id, compid, status, buildid, buildurl, dockersha, dockertag, gitcommit, gitrepo, giturl) \ + VALUES {}'.format(records_list_template) + + #Retry logic for failed query + no_of_retry = db_conn_retry + attempt = 1; + while True: + try: + with engine.connect() as connection: + conn = connection.connection + cursor = conn.cursor() + cursor.execute(sql, data_list) + # commit the changes to the database + rows_inserted = cursor.rowcount + # Commit the changes to the database + conn.commit() + conn.close() + + if rows_inserted > 0: + response.status_code = status.HTTP_201_CREATED + return {"message": 'components created succesfully'} + + response.status_code = status.HTTP_200_OK + return {"message": 'components not created'} + + except (InterfaceError, OperationalError) as ex: + if attempt < no_of_retry: + logging.error( + "Database connection error: {} - sleeping for {}s" + " and will retry (attempt #{} of {})".format( + ex, sleep_for, attempt, no_of_retry + ) + ) + #200ms of sleep time in cons. retry calls + sleep(0.2) + attempt += 1 + continue + else: + raise + except HTTPException: raise except Exception as err: @@ -294,22 +338,43 @@ async def delete_compitem(request: Request, compid: int): raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed:" + str(err)) from None try: - with engine.connect() as connection: - conn = connection.connection - cursor = conn.cursor() - - sql1 = "DELETE from dm.dm_compitemprops where compitemid in (select id from dm.dm_componentitem where compid = " + str(compid) + ")" - sql2 = "DELETE from dm.dm_componentitem where compid=" + str(compid) - rows_deleted = 0 - cursor.execute(sql1) - cursor.execute(sql2) - rows_deleted = cursor.rowcount - # Commit the changes to the database - conn.commit() - - # response.status_code = status.HTTP_200_OK - return {"message": 'component deleted succesfully'} - + + #Retry logic for failed query + no_of_retry = db_conn_retry + attempt = 1; + while True: + try: + with engine.connect() as connection: + conn = connection.connection + cursor = conn.cursor() + + sql1 = "DELETE from dm.dm_compitemprops where compitemid in (select id from dm.dm_componentitem where compid = " + str(compid) + ")" + sql2 = "DELETE from dm.dm_componentitem where compid=" + str(compid) + rows_deleted = 0 + cursor.execute(sql1) + cursor.execute(sql2) + rows_deleted = cursor.rowcount + # Commit the changes to the database + conn.commit() + + # response.status_code = status.HTTP_200_OK + return {"message": 'component deleted succesfully'} + + except (InterfaceError, OperationalError) as ex: + if attempt < no_of_retry: + logging.error( + "Database connection error: {} - sleeping for {}s" + " and will retry (attempt #{} of {})".format( + ex, sleep_for, attempt, no_of_retry + ) + ) + #200ms of sleep time in cons. retry calls + sleep(0.2) + attempt += 1 + continue + else: + raise + except HTTPException: raise except Exception as err: @@ -350,29 +415,50 @@ async def update_compitem(request: Request, compitemList: List[CompItemModel]): raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Authorization Failed:" + str(err)) from None try: + data_list = [] for col in compitemList: row = (col.compid, col.status, col.buildid, col.buildurl, col.dockersha, col.dockertag, col.gitcommit, col.gitrepo, col.giturl, col.id) # this will be changed data_list.append(row) - - with engine.connect() as connection: - conn = connection.connection - cursor = conn.cursor() - # # execute the INSERT statement - # records_list_template = ','.join(['%s'] * len(data_list)) - sql = 'UPDATE dm.dm_componentitem set compid=%s, status=%s, buildid=%s, buildurl=%s, dockersha=%s, dockertag=%s, gitcommit=%s, gitrepo=%s, giturl=%s \ - WHERE id = %s' - cursor.executemany(sql, data_list) - # commit the changes to the database - rows_inserted = cursor.rowcount - # Commit the changes to the database - conn.commit() - if rows_inserted > 0: - return {"message": 'components updated succesfully'} + #Retry logic for failed query + no_of_retry = db_conn_retry + attempt = 1; + while True: + try: + with engine.connect() as connection: + conn = connection.connection + cursor = conn.cursor() + # # execute the INSERT statement + # records_list_template = ','.join(['%s'] * len(data_list)) + sql = 'UPDATE dm.dm_componentitem set compid=%s, status=%s, buildid=%s, buildurl=%s, dockersha=%s, dockertag=%s, gitcommit=%s, gitrepo=%s, giturl=%s \ + WHERE id = %s' + cursor.executemany(sql, data_list) + # commit the changes to the database + rows_inserted = cursor.rowcount + # Commit the changes to the database + conn.commit() + + if rows_inserted > 0: + return {"message": 'components updated succesfully'} - return {"message": 'components not updated'} - + return {"message": 'components not updated'} + + except (InterfaceError, OperationalError) as ex: + if attempt < no_of_retry: + logging.error( + "Database connection error: {} - sleeping for {}s" + " and will retry (attempt #{} of {})".format( + ex, sleep_for, attempt, no_of_retry + ) + ) + #200ms of sleep time in cons. retry calls + sleep(0.2) + attempt += 1 + continue + else: + raise + except Exception as err: print(err) raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(err)) from None From 9807d8323b5363f413e9aa93c13be2da965ef7b2 Mon Sep 17 00:00:00 2001 From: "dependency-ai-bot[bot]" <207322259+dependency-ai-bot[bot]@users.noreply.github.com> Date: Sat, 12 Apr 2025 10:52:54 +0000 Subject: [PATCH 9/9] Update dependencies in requirements.txt --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 07002d48..b4d72822 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # psycopg2==2.8.6 - installed by base image #sqlalchemy - installed by base image #fastapi - installed by base image -uvicorn==0.15.0 -pydantic==1.8.2 -requests==2.25.0 \ No newline at end of file +uvicorn==0.34.0 +pydantic==2.11.3 +requests==2.32.3 \ No newline at end of file