From 3fcdbcafdde55c283955f1c428279019f5d3ef6a Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Fri, 22 Jun 2018 16:46:43 -0500 Subject: [PATCH 1/8] Deploy to a server at HCC via SFTP --- .travis.yml | 37 +++++++++++++++--------- travis-ci/id_gctuploader.enc | Bin 0 -> 1680 bytes travis-ci/id_gctuploader.pub | 1 + travis-ci/upload_source_tarballs.sh | 43 ++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 13 deletions(-) create mode 100644 travis-ci/id_gctuploader.enc create mode 100644 travis-ci/id_gctuploader.pub create mode 100755 travis-ci/upload_source_tarballs.sh diff --git a/.travis.yml b/.travis.yml index f1b2a25b1d..b5d0186d14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,18 +50,29 @@ jobs: script: - travis-ci/setup_tasks.sh deploy: - on: - tags: true - all_branches: false - repo: gridcf/gct - skip_cleanup: true - file_glob: true - provider: releases - overwrite: true - # ^ this doesn't actually work but I'm leaving it in there in - # case it gets fixed later - api_key: - secure: gNgkiuvPm/7eEKa203yXe1jPwBmALWLXPpXhfNZO7wBISPgB1XZHgRNMr6DKlE5gucElKSjWWhQbn5VCfJm3OB+KCb1Tx1Xsw6xmsLrEL0d+YF/cqQl98a4ZRQjrRW4CBHz6V/FGWYiI53YDvwb/wfSC3uhIXlBjN+CeLv2aJQuOYZ9ZJHVDZYKWp2yrL5qvCvwXVZfNZNCf1jNztKFyDyQ5xCT7ePjQAbASkkvO1ZMRVgwloiKqZVWINhjcYlKaTI5ov0U0yPLneE2ybZz5mxFCNSa4okQf3gb36eMge7i7yulk8R7uVxsFte0bhLEslGnYn+dtLT+4RmYflt5OqFs3z0fVAG+q9kwsLii50iE1pmY9QHt+P+8vkajX7T+QiK+qers41kmDvArFZboIDMkvHWaNceAtDZ9e/Pd54hL8ztJUjSa3FPqz6mwa6og3RNd4GFiER4aO6ISfdshCxTRkUnYMetzI+p77cP4HI2I87grJGOJUIGpp9qzukvPAGVS79kmBgV7Ykd7GQccKiiNx25hbACV81pZ9WZ9MhYVoc/SkzrST7b9Tba8QHCrtdZWafpN1awTkbjOxw4vN8XIaexE7WS6d7TduhTORb6awcABFzL9TNtQKQJWjlLsRv4VKVNz59mKrux/VnfF8vkMEqdn/P8d6UCn7oVe6i+g= - file: travis_deploy/* + - provider: releases + on: + tags: true + all_branches: false + repo: gridcf/gct + skip_cleanup: true + file_glob: true + overwrite: true + # ^ this doesn't actually work but I'm leaving it in there in + # case it gets fixed later + api_key: + secure: gNgkiuvPm/7eEKa203yXe1jPwBmALWLXPpXhfNZO7wBISPgB1XZHgRNMr6DKlE5gucElKSjWWhQbn5VCfJm3OB+KCb1Tx1Xsw6xmsLrEL0d+YF/cqQl98a4ZRQjrRW4CBHz6V/FGWYiI53YDvwb/wfSC3uhIXlBjN+CeLv2aJQuOYZ9ZJHVDZYKWp2yrL5qvCvwXVZfNZNCf1jNztKFyDyQ5xCT7ePjQAbASkkvO1ZMRVgwloiKqZVWINhjcYlKaTI5ov0U0yPLneE2ybZz5mxFCNSa4okQf3gb36eMge7i7yulk8R7uVxsFte0bhLEslGnYn+dtLT+4RmYflt5OqFs3z0fVAG+q9kwsLii50iE1pmY9QHt+P+8vkajX7T+QiK+qers41kmDvArFZboIDMkvHWaNceAtDZ9e/Pd54hL8ztJUjSa3FPqz6mwa6og3RNd4GFiER4aO6ISfdshCxTRkUnYMetzI+p77cP4HI2I87grJGOJUIGpp9qzukvPAGVS79kmBgV7Ykd7GQccKiiNx25hbACV81pZ9WZ9MhYVoc/SkzrST7b9Tba8QHCrtdZWafpN1awTkbjOxw4vN8XIaexE7WS6d7TduhTORb6awcABFzL9TNtQKQJWjlLsRv4VKVNz59mKrux/VnfF8vkMEqdn/P8d6UCn7oVe6i+g= + file: travis_deploy/* + - provider: script + on: + #tags: true + tags: false + #all_branches: false + all_branches: true + #repo: gridcf/gct + repo: matyasselmeci/gct + skip_cleanup: true + script: bash travis-ci/upload_source_tarballs.sh + # vim:ft=yaml:sw=2:sts=2:et diff --git a/travis-ci/id_gctuploader.enc b/travis-ci/id_gctuploader.enc new file mode 100644 index 0000000000000000000000000000000000000000..0688cc104cd7bcff86666496ea681c595f898522 GIT binary patch literal 1680 zcmV;B25vfCvsY)p{FtAM<5PH+ddH7p zL*cn+nCG&n@lpugOJcVFbl(G6bq&P_3u6t&4wl+VF@+=xh^XQuKzIta>cA0XJXIXx zJ*?VU{p?4D}vsAfh1&0Nhuo14rTHL<>qGY5JQFKnFC4q zX|sOI#I{l;I5-f_&ReeH1^FEsm#8IBBbOwyURuE34O-IHHkB&G3xL!9q{Xo3XjrHy zh#S1T_y%A=lif?o5XYWxoCC`ja2cw~7&&Nji4i4ms+WrQ{d-&7UBEU7!#{T>teJk7 zrA!xiepl?A=o}V`g&5kU*D@SU+RHkZbK*fmKO_B}? zWnQf2+kwwj^JL09*0{RGGJ_>VCSbnj{YkJ^=xeRHj2*kDsE5^2JnKC#$iZmL}A*MZGYD)$XEDMs={f`x|Cx!7%v z$mo$n+g{^A2iDfnF27(5W=5VV!BCs$$hW3-H544QBaU3Hxw6iV%5P3_Fj%{E20eG) zyST$LYJ5(HRbV~59Usqx5v@#P@C?8|CBQm>;TI7Qyz++)f9b|;{)@{V(R<$hCLIye zS@|iy*^PGv_HS<1b+Ix(Zm_pWb)jkS3UDFT_09}>posBVwOG<7%pLhk(@`g7Z?JpH zFE8CDVsd^92v~V1U0byca-w?lAyi?5Yo=*OX1?a!yA$}OKe($ycjr_06^@Ir_?58c zRA;JCmtE}kytN;)#vVZGR6U_sL}A=b(2z(K+&8FY%Tpft%S|c(w<~d(eHYEYa?Lpr zepH#G;F)z?cvi)%CLY9b{q>Jk8#06{pk_#3m6ncomIuZRQnamBY2}h)8P^nW{cvo7 zdK?TDBX>@oDksrcJuf>;%8PAC>ElI1r89l}8R+Aod=lm_!^w47$e;-D1rK#=w&mxN zEBc00-cM%;o4PEX&gX|TA*|IpW=lwsQ4;NOlSz@$U}O8kSbR!QRUy^BJa>1o6iOt0v@lk*3~Qf~TAOk*mq@QK9#2Vk4NI8aL>6|lC} zV-EMzbgQhz`EFXgW}e62aU=jI;x%bmp{~8R?4Zg(RvRI*G%!_nEF-0SE{yXSw_-W@ zl5ugf59$c`)LYZo;rKlIm1Iq5FqvZ;lK`86KVZ3+npG~*B_nAx?>PZ;eb5+A)`G%K z6;IC(Fh=o&-%Xqtk-KQr7Ky6!&e=!Vya;qI4DJ7|WaSb?;ytI-OZlMfppXopb+K0R aRjxJREZ~ns8#6@}2_OHs;&7|XJ$agW{!@hj literal 0 HcmV?d00001 diff --git a/travis-ci/id_gctuploader.pub b/travis-ci/id_gctuploader.pub new file mode 100644 index 0000000000..ecd1e8772b --- /dev/null +++ b/travis-ci/id_gctuploader.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwudTBpxkoqsfndMVVyoaDe8CBpm+/fYSaJgQoi8PJW3AYM8r9wdhBkan2rnzWIYFfj7QxFLs1ekEtSv0wCspQ9wpsuPmPjLaSBI6foTUWcDepz4BOiysit3hOqocdWy7bp6kF8cbnUdzhDfVxnbSWHOTBVRiR/M2htWzdSRxxwqdY5lyrLC+MObUPLSvkj1RzF2KzsuXKnDr31l0eugmhIj4qBOFIDb2nq4otbZ0vFbT1R4N7LetyOpnrDZ/Ps80GTFR77qdIhe05mY/inVBR6OQ8Yw/fmY2DPIcPYYXepXdNZx/ZK2qzsfk6xdfBglsBNSxyFNU42ndlGTteyIPx gctuploader (created by matyas@cs.wisc.edu) diff --git a/travis-ci/upload_source_tarballs.sh b/travis-ci/upload_source_tarballs.sh new file mode 100755 index 0000000000..282f6e3a96 --- /dev/null +++ b/travis-ci/upload_source_tarballs.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Upload source tarballs in the "package-output/" directory in the repo +# to a remote host, via SCP. + +set -eux + +keyfile=$(pwd -P)/travis-ci/id_gctuploader + +# obtained by running "ssh-keyscan hcc-osg-software2.unl.edu" +hostsig="hcc-osg-software2.unl.edu ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2AIWAVx2KY+GhDab9SdxLTvjjzTiNa4pfHe7TvRZ5O+qZNc4c8sBlsG7OZGZvDLMRjGTKFyjJx3jDVUwaf14DwzQi9rgZxEZgBsRFffLATZqz+DyVN1H9uw215pah9Wh6yzaqMn51y6kqg0kk/ip62cYcXFgLKUNkzV0yz5WFugm5ziROZn01v5o74VdCABTAdlZhviUoObCn+bycXoUGGETY5GZ3muAW6y5LydDTD+2S97qJWGdSW7JBIfcmU7n5dl8MrtYKYwGswOgdUDrLtCp6CdZt/Evr+3NyLp35IhLnwxdkBBlKHPY0jXrGHyemsXa0Hq0PG/Ih5d0M8RMp" + + +echo "$hostsig" > ~/.ssh/known_hosts + +( + umask 077 + mkdir -p ~/.ssh + openssl aes-256-cbc \ + -K $encrypted_677f6546cb93_key \ + -iv $encrypted_677f6546cb93_iv \ + -in "$keyfile.enc" -out "$keyfile" \ + -d +) + +root=$(git rev-parse --show-toplevel) +cd "$root" + +cd package-output +sha512sum *.tar.gz > sha512sums + +sftp \ + -o "PubkeyAuthentication=yes" \ + -o "IdentitiesOnly=yes" \ + -i "$keyfile" -b - gctuploader@hcc-osg-software2.unl.edu <<__END__ +-mkdir gct6 +cd gct6 +-mkdir sources +cd sources +put *.tar.gz +put sha512sums +__END__ +# vim:et:sts=4:sw=4 From b229ffd112cbd86e7bda43eaee81fc41662c9a98 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Fri, 17 Aug 2018 15:13:10 -0500 Subject: [PATCH 2/8] Encrypt gctuploader key for gridcf/gct repo --- .travis.yml | 9 +++++---- travis-ci/id_gctuploader.enc.gridcf | Bin 0 -> 1680 bytes ...ader.enc => id_gctuploader.enc.matyasselmeci} | Bin travis-ci/upload_source_tarballs.sh | 4 +++- 4 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 travis-ci/id_gctuploader.enc.gridcf rename travis-ci/{id_gctuploader.enc => id_gctuploader.enc.matyasselmeci} (100%) diff --git a/.travis.yml b/.travis.yml index b5d0186d14..74d44b48eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,14 +65,15 @@ jobs: file: travis_deploy/* - provider: script on: - #tags: true + # tags: true tags: false - #all_branches: false + # all_branches: false all_branches: true - #repo: gridcf/gct + # repo: gridcf/gct repo: matyasselmeci/gct skip_cleanup: true - script: bash travis-ci/upload_source_tarballs.sh + # script: bash travis-ci/upload_source_tarballs.sh gridcf + script: bash travis-ci/upload_source_tarballs.sh matyasselmeci # vim:ft=yaml:sw=2:sts=2:et diff --git a/travis-ci/id_gctuploader.enc.gridcf b/travis-ci/id_gctuploader.enc.gridcf new file mode 100644 index 0000000000000000000000000000000000000000..0b17590379c71ec6281b72baf84802147aa6a946 GIT binary patch literal 1680 zcmV;B25rQQ(>)|Ijfqk$? z3OdcqbcSZJROK}}%4M@b9}ucu$;vzrMhfIML&w+@@$!oTVLqHQ^OEd)%8~PANfVX{ zV08|aTPjW}{Ty=vJ#I0cJ2+ifEcpXn-<$|uNkvc@1!bCg{*imE;e!8Mf~A!Bv(+ca)vEj{Bu~6%&PD9W9>|j6P|pR9!Mc!jY-vtyisRigU32 zxnEO8{>OZwJ(ZH{SbGzCq>t5mfPY8#ocV?%rc z9sA7PrI2Pn=0@+^YOOugY(-(dFa+P2%x-?FHYit^7E$x6yBkzRw8h=7y!oit7rVNY5ybh$4TdK~bPZ%KtF|X6)l#%>CVXef0=lfA^ zcnygn!H$_Q7XuLGZkl>0{{Mn;UoOUW?4;$Gk#W@)Hh-N}>vz_0Ud;dInyjN+W6iix zH+Th7s)dNFUp}JWabce$=T0uQUdw5(&_L<8x!8rtgANV<29!cRtZ`` z5~AQhQceThw~Np4_FcFQcTwHRWfT5EGT{W;K9U zbDz(9(a3A9^0%gINd5YRnI^9TcGqvj1N*3*)c)X1=pi#1)W|r6#^(;)Zuq#jb5Mpn zuT*S!(Lu0Gf7xiYTiODxU`P0(%f-g29Nh?;u*2OT7b&#Mi73i|yCu@R(0*O2WtEO5 zCucZif6qOrEJ>&n{lQicQ%IDJTAKlRUQi9SHHk|7rFf~NgGqHLbyZ`0mkID01PyT$ z>Qx=!@mPK*)(Gp$)@wV9@9q0*8Wev!YRy?Q zkbMJ;JAEMiIGd-pcB=c7$%(1>mWHlvM88Us3Sqx(HA&4g!SpuGPra{FrJ!Hh>dpv2 z$dq3Ey6$d=-m9dL;7~5Hjs%G0FouRl9on_8G71+ z0bmO{ao4-v7E6v#6=FsQ{z|q0#&nbf*@5yq-M9cVfqP6mF5+BH^;yZ^Itl$;ALDmY zhgf8!ISJktvt!F`IDQRwGT-T^CJ#e>Gt80gIkPKBX7>5l>tw!`K@& zkp71HOe~!KYHTQ^n~tSbiW|GsvoGhIwgq5O2Nz_Le|uJmEKL@ffu z3S`&8QU(~&Ix&)-I_Tj&zQ|LWbi3>EQ)WNXLLSvh(wtw6G<3-0Z|WH9>~#~g^RJoM z%sMiNB@`=VSvY+vsHLr2Q@Q4Os;=k>S4T_~>7XMaLG9$jyAES2Qab^N>t}~B@p2i- z{U92*_n=_2CdVBqCW7!!!{jvDDgC$RbstdI zlaAd39wlYO79VMM9MET56rqP*Nv$neq{iy7Iux0GRqu}WQXR$wR5aLHuLY>n$rnLc z;D84#Os{P#i>zwOD>b_@u;rVryvb#ybJnXQL3t_XxAv1IBztRLtEqniXsA?=Vq#;s z%%<6IJiabi)vYV&uR?2~Sa2L1ol+qZ6LOdB(Z@L*ccB0Vy*aorC;gSK$7|q&g%*VJ z>#UC@)Qj@2u9A9N@l|bjh5^?Kjmu!8VO*Fi+Oe=Gr%7T1jn76?*0Pi|K!1Se>M*Bm aS-Qn!_&S2RZ|~KfIgZ3aAg%Gotz0ojbVvjM literal 0 HcmV?d00001 diff --git a/travis-ci/id_gctuploader.enc b/travis-ci/id_gctuploader.enc.matyasselmeci similarity index 100% rename from travis-ci/id_gctuploader.enc rename to travis-ci/id_gctuploader.enc.matyasselmeci diff --git a/travis-ci/upload_source_tarballs.sh b/travis-ci/upload_source_tarballs.sh index 282f6e3a96..395af610e7 100755 --- a/travis-ci/upload_source_tarballs.sh +++ b/travis-ci/upload_source_tarballs.sh @@ -5,6 +5,8 @@ set -eux +repo_owner=$1 + keyfile=$(pwd -P)/travis-ci/id_gctuploader # obtained by running "ssh-keyscan hcc-osg-software2.unl.edu" @@ -19,7 +21,7 @@ echo "$hostsig" > ~/.ssh/known_hosts openssl aes-256-cbc \ -K $encrypted_677f6546cb93_key \ -iv $encrypted_677f6546cb93_iv \ - -in "$keyfile.enc" -out "$keyfile" \ + -in "$keyfile.enc.$repo_owner" -out "$keyfile" \ -d ) From b691ac7df88ce8b6ab7ed38c8cc01c22a4867c95 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Mon, 27 Aug 2018 16:21:51 -0500 Subject: [PATCH 3/8] Elaborate on why hcc-osg-software2.unl.edu --- travis-ci/upload_source_tarballs.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/travis-ci/upload_source_tarballs.sh b/travis-ci/upload_source_tarballs.sh index 395af610e7..75dc625dfe 100755 --- a/travis-ci/upload_source_tarballs.sh +++ b/travis-ci/upload_source_tarballs.sh @@ -9,6 +9,9 @@ repo_owner=$1 keyfile=$(pwd -P)/travis-ci/id_gctuploader +# repo.gridcf.org is an alias for this: +upload_server=hcc-osg-software2.unl.edu + # obtained by running "ssh-keyscan hcc-osg-software2.unl.edu" hostsig="hcc-osg-software2.unl.edu ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2AIWAVx2KY+GhDab9SdxLTvjjzTiNa4pfHe7TvRZ5O+qZNc4c8sBlsG7OZGZvDLMRjGTKFyjJx3jDVUwaf14DwzQi9rgZxEZgBsRFffLATZqz+DyVN1H9uw215pah9Wh6yzaqMn51y6kqg0kk/ip62cYcXFgLKUNkzV0yz5WFugm5ziROZn01v5o74VdCABTAdlZhviUoObCn+bycXoUGGETY5GZ3muAW6y5LydDTD+2S97qJWGdSW7JBIfcmU7n5dl8MrtYKYwGswOgdUDrLtCp6CdZt/Evr+3NyLp35IhLnwxdkBBlKHPY0jXrGHyemsXa0Hq0PG/Ih5d0M8RMp" @@ -34,7 +37,7 @@ sha512sum *.tar.gz > sha512sums sftp \ -o "PubkeyAuthentication=yes" \ -o "IdentitiesOnly=yes" \ - -i "$keyfile" -b - gctuploader@hcc-osg-software2.unl.edu <<__END__ + -i "$keyfile" -b - gctuploader@$upload_server <<__END__ -mkdir gct6 cd gct6 -mkdir sources From a41403cc6b3079477b313bb8bc298021181412cd Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Mon, 27 Aug 2018 16:33:41 -0500 Subject: [PATCH 4/8] Don't upload gct-*.tar.gz; don't need it plus has a timestamp in the name so a new one would always get uploaded --- travis-ci/upload_source_tarballs.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/travis-ci/upload_source_tarballs.sh b/travis-ci/upload_source_tarballs.sh index 75dc625dfe..f28b5e5add 100755 --- a/travis-ci/upload_source_tarballs.sh +++ b/travis-ci/upload_source_tarballs.sh @@ -32,6 +32,10 @@ root=$(git rev-parse --show-toplevel) cd "$root" cd package-output +rm -f gct-*.tar.gz +# ^ has a timestamp in the name so always gets updated whether anything changed +# or not. Between the git repo and the tarballs for the individual packages, +# this is unnecessary anyway. sha512sum *.tar.gz > sha512sums sftp \ From 917bbd0904ed4cce35641539fb8edf84630e6b25 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Tue, 28 Aug 2018 13:48:11 -0500 Subject: [PATCH 5/8] Upload on releases only (i.e. tagged versions on the gridcf/gct repo) --- .travis.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 74d44b48eb..ea9390295d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,15 +65,11 @@ jobs: file: travis_deploy/* - provider: script on: - # tags: true - tags: false - # all_branches: false - all_branches: true - # repo: gridcf/gct - repo: matyasselmeci/gct + tags: true + all_branches: false + repo: gridcf/gct skip_cleanup: true - # script: bash travis-ci/upload_source_tarballs.sh gridcf - script: bash travis-ci/upload_source_tarballs.sh matyasselmeci + script: bash travis-ci/upload_source_tarballs.sh gridcf # vim:ft=yaml:sw=2:sts=2:et From a7045eb0f935ab0cc23d39f386b8e2236749f353 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Tue, 28 Aug 2018 17:02:17 -0500 Subject: [PATCH 6/8] Sanity check the tarballs --- travis-ci/make_source_tarballs.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/travis-ci/make_source_tarballs.sh b/travis-ci/make_source_tarballs.sh index 7ddef5bca6..2236bf28cc 100755 --- a/travis-ci/make_source_tarballs.sh +++ b/travis-ci/make_source_tarballs.sh @@ -48,5 +48,27 @@ echo '========================================================================== pushd "$root/myproxy/oauth/source" time python setup.py sdist mv dist/*.tar.gz "$root/package-output/" + +err=0 +pushd "$root/package-output/" +for tb in *.tar.gz; do + if [[ ! -s $tb ]]; then + echo "$tb is empty!" + err=1 + else + filetype=$(file -bzi "$tb") + if [[ $filetype != *application/x-tar* ]]; then + echo "$tb: unexpected file type '$filetype'" + err=1 + fi + fi +done +popd + +if [[ $err -ne 0 ]]; then + echo "Sanity check failed -- bailing" + exit $err +fi + popd From 20e2f8df1a480e5e31d1b6b55428cee417ecf325 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Wed, 29 Aug 2018 11:01:57 -0500 Subject: [PATCH 7/8] Upload tarballs only if no files by that name exist remotely --- .travis.yml | 1 + travis-ci/upload_source_tarballs.sh | 32 +++++++++++++++++++++-------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index ea9390295d..cebc1c1fab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,6 +39,7 @@ jobs: env: - IMAGE=centos:centos7 TASK=srpms sudo: required + skip_cleanup: true services: - docker diff --git a/travis-ci/upload_source_tarballs.sh b/travis-ci/upload_source_tarballs.sh index f28b5e5add..15abca1ad5 100755 --- a/travis-ci/upload_source_tarballs.sh +++ b/travis-ci/upload_source_tarballs.sh @@ -17,6 +17,13 @@ hostsig="hcc-osg-software2.unl.edu ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC2AIWAV echo "$hostsig" > ~/.ssh/known_hosts +cat > ~/.ssh/config <<__END__ +Host $upload_server +User gctuploader +IdentityFile $keyfile +PubkeyAuthentication yes +IdentitiesOnly yes +__END__ ( umask 077 @@ -36,17 +43,26 @@ rm -f gct-*.tar.gz # ^ has a timestamp in the name so always gets updated whether anything changed # or not. Between the git repo and the tarballs for the individual packages, # this is unnecessary anyway. -sha512sum *.tar.gz > sha512sums -sftp \ - -o "PubkeyAuthentication=yes" \ - -o "IdentitiesOnly=yes" \ - -i "$keyfile" -b - gctuploader@$upload_server <<__END__ +sftp -b - $upload_server &>/dev/null <<__END__ -mkdir gct6 cd gct6 -mkdir sources -cd sources -put *.tar.gz -put sha512sums __END__ + +# Create individual checksum files instead of one big one because we want to +# keep checksums for old tarballs that are already in the repo. +for tarball in *.tar.gz; do + # Don't upload the tarball if it already exists + tbpath=gct6/sources/$tarball + escaped_tbpath=$(sed -e 's#\.#\\.#g' <<<"$tbpath") + if ! sftp -b - $upload_server <<<"ls $tbpath" | grep -qx "$escaped_tbpath\s*"; then + sha512sum "$tarball" > "$tarball.sha512" + sftp -b - $upload_server <<__END__ +put "$tarball" "$tbpath" +put "$tarball.sha512" "$tbpath.sha512" +__END__ + fi +done + # vim:et:sts=4:sw=4 From 78a4644126809136489bcc7657a40e87a3660ce6 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Thu, 6 Sep 2018 11:23:02 -0500 Subject: [PATCH 8/8] Check that the remote tarball is OK and upload if not --- travis-ci/upload_source_tarballs.sh | 56 +++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/travis-ci/upload_source_tarballs.sh b/travis-ci/upload_source_tarballs.sh index 15abca1ad5..bc44992f2f 100755 --- a/travis-ci/upload_source_tarballs.sh +++ b/travis-ci/upload_source_tarballs.sh @@ -50,19 +50,63 @@ cd gct6 -mkdir sources __END__ + +remote_tarball_ok () { + local tarball remote_tarball + tarball=$1 + remote_tarball=$2 + + local tmp=$(mktemp -d) + trap "rm -rf $tmp" RETURN + # Check for file and sha512sum existence. Download the file so we + # can run extra tests. + sftp -b - $upload_server <<__END__ +get $remote_tarball $tmp/$tarball +get $remote_tarball.sha512 $tmp/$tarball.sha512 +__END__ + if [[ $? -ne 0 ]]; then + echo "***** Couldn't download remote tarball $tarball and/or checksum ******" + return 1 + fi + + # Check that the downloaded sha512sum matches the downloaded file. + pushd $tmp + sha512sum --quiet -c $tarball.sha512 + local ret=$? + popd + if [[ $ret -ne 0 ]]; then + echo "****** Remote tarball $tarball checksum doesn't match ******" + return 1 + fi + + # Check that the downloaded file is a tarball. + filetype=$(file -bzi "$tmp/$tarball") + if [[ $filetype != *application/x-tar* ]]; then + echo "****** Remote tarball $tarball doesn't look like a tarball ******" + return 1 + fi + + return 0 +} + + # Create individual checksum files instead of one big one because we want to # keep checksums for old tarballs that are already in the repo. +set +e for tarball in *.tar.gz; do - # Don't upload the tarball if it already exists - tbpath=gct6/sources/$tarball - escaped_tbpath=$(sed -e 's#\.#\\.#g' <<<"$tbpath") - if ! sftp -b - $upload_server <<<"ls $tbpath" | grep -qx "$escaped_tbpath\s*"; then + remote_tarball=gct6/sources/$tarball + # Don't upload the tarball if it already exists and is valid + if remote_tarball_ok "$tarball" "$remote_tarball"; then + echo "****** Remote tarball $tarball OK... leaving remote as-is ******" + else + echo "****** Checksumming and uploading $tarball ******" sha512sum "$tarball" > "$tarball.sha512" sftp -b - $upload_server <<__END__ -put "$tarball" "$tbpath" -put "$tarball.sha512" "$tbpath.sha512" +put "$tarball" "$remote_tarball" +put "$tarball.sha512" "$remote_tarball.sha512" __END__ fi done +set -e # vim:et:sts=4:sw=4