ci_guard

#!/bin/bash
# ./ci_guard.sh

ERRORSTATUS=10
TESTSTATUS=0
USER="moonx"
IPs="192.168.1.102 " # add host ip like "xx.xx.xx.xx xx.xx.xx.xx", delimit by space
dic=([6]="192.168.1.102" [7]="xxx.xxx.xxx.xxx")
webhookUri="https://hooks.slack.com/services/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"
lastrestart=`date +%s`
restart_gitlab_runner(){
   for IP in ${IPs}
   do
       echo "restart gitlab-runner on ${IP}: ssh ${USER}@${IP}"
       ssh ${USER}@${IP} "sudo service gitlab-runner restart; exit"
       #ssh ${USER}@${IP} "date >> /usr/download/testcov/log; exit"
       sleep 3
   done
}

sendmsgtoslack(){
    runnerip=""
    str=$*
    arr=(${str// / })  
    for i in ${arr[@]}  
    do  
        runnerip+=${dic[i]}" "
    done
    url="https://hooks.slack.com/services/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"
    curdate="$(date "+%Y_%m_%d %H:%M:%S")"
    #curl -v POST -H 'Content-type: application/json' -d '{
    curl POST -H 'Content-type: application/json' -d '{
      "response_type": "ephemeral",
      "channel": "#demo",
      "text": "CI Bug Report",
      "username": "Bug News",
      "icon_emoji": "",
      "attachments": [{
        "color": "#a6364f",
        "title": "CI crashs.",
        "title_link": "https://gitlab.moonx.cn/Developers/apollo/pipelines",
        "text": "'"${curdate}"'",
        "fields": [{
          "title": "'"${runnerip}"'",
        }]
      }]
    }' $url
}

main(){
    echo 'begin guard the ci.'
    while true
    do
    # get pipelines info
    pipelinesinfo=`curl --header "PRIVATE-TOKEN: UsDsmqSpivXMmNoCHcci" 
               "https://gitlab.moonx.cn/api/v4/projects/2/pipelines" --insecure` # |python -m json.tool`
    runnersinfo=`curl --header "PRIVATE-TOKEN: UsDsmqSpivXMmNoCHcci" 
             "https://gitlab.moonx.cn/api/v4/projects/2/runners" --insecure`

    # resolve pipelines info
    x=`python checkcistatus.py $pipelinesinfo $runnersinfo`
    ci_status=$?
    echo "python result:"$x

    if [ $ERRORSTATUS == $ci_status ]
    then   
       restart_gitlab_runner
       tmptime=`date -d "-10 min" +%s`
       if [ $tmptime -lt $lastrestart ]
       then
           sendmsgtoslack $x
       fi
       lastrestart=`date +%s`
    fi
    #sendmsgtoslack $x
    sleep 300
    done
    echo 'exit guard the ci.'
}

main

# coding=utf-8
# python checkcistatus.py $pipelinesinfo $runnersinfo

import sys
import re

ERRORSTATUS=10

pipeidstr='"id":'
pendingpipe='"status":"pending"'
runningpipe='"status":"running"'

template_str_runner='"name":"gitlab-runner"'
template_str_active_runner='"active":true'

if __name__ == "__main__":
    if len(sys.argv) < 3:
        print "usage: python checkcistatus.py $pipelinesinfo $runnersinfo"
        sys.exit(1)

#for i in range(0,len(sys.argv)):
#    print(i, sys.argv[i])
pipe_num=len(re.findall(pipeidstr, sys.argv[1]))
runningpipe_num=len(re.findall(runningpipe, sys.argv[1]))
pendingpipe_num=len(re.findall(pendingpipe, sys.argv[1]))
#print "pipe_num:%d,running:%d,pending:%d" % (pipe_num,runningpipe_num,pendingpipe_num)

runner_num=len(re.findall(template_str_runner, sys.argv[2]))
active_runner_num=len(re.findall(template_str_active_runner, sys.argv[2]))
reg = re.compile('{"id":([^,]*),"description":"([^,]*)","active":([^,]*),"is_shared":([^,]*),"name":"([^,]*)","online":([^,]*),"status":"([^,]*)"}')
runners = re.findall(reg,sys.argv[2])
for runner in runners:
    #print runner
    if runner[6] == 'online':
       print runner[0]
#print "runner:%d,active_runner:%d
" % (runner_num,active_runner_num)

if 0==runningpipe_num and 0!=pendingpipe_num:
    #print "[error]no running pipeline, pending pipeline exists."
    #print "[error]need to restart gitlab-runner."
    sys.exit(ERRORSTATUS)

if 0!=pendingpipe_num and runningpipeline_num < active_runner_num:
    #print "[error]running pipeline less than active runner, pending pipeline exists."
    #print "[error]need to restart gitlab-runner."
    sys.exit(ERRORSTATUS) 

#print "[info]not need to restart gitlab-runner."
sys.exit(0)

#coding:utf-8
#server.login(my_sender,"ydydmvthorlqbgbg") my_sender='490195356@qq.com'
#coding=utf-8
import smtplib
from email.mime.text import MIMEText
msg_from='490195356@qq.com'
passwd='ydydmvthorlqbgbg'
msg_to="490195356@qq.com,peng.yang@moonx.ai"
subject="no reply<python邮件测试>"    
content='''
#coding:utf-8
#server.login(my_sender,"ydydmvthorlqbgbg") my_sender='490195356@qq.com'
#coding=utf-8
import smtplib
from email.mime.text import MIMEText
msg_from='490195356@qq.com'
passwd='ydydmvthorlqbgbg'
msg_to="490195356@qq.com,peng.yang@moonx.ai"
subject="no reply<python邮件测试>"    
content="pp"
msg = MIMEText(content)
msg['Subject'] = subject
msg['From'] = msg_from
msg['To'] = msg_to
try:
    s = smtplib.SMTP_SSL("smtp.qq.com",465)
    s.login(msg_from, passwd)
    s.sendmail(msg_from, msg_to.split(','), msg.as_string())
    print "发送成功"
except s.SMTPException,e:
    print "发送失败"
finally:
    s.quit()  
'''
msg = MIMEText(content)
msg['Subject'] = subject
msg['From'] = msg_from
msg['To'] = msg_to
try:
    s = smtplib.SMTP_SSL("smtp.qq.com",465)
    s.login(msg_from, passwd)
    s.sendmail(msg_from, msg_to.split(','), msg.as_string())
    print "发送成功"
except s.SMTPException,e:
    print "发送失败"
finally:
    s.quit()

gitlab + gitlab-ci + slack 持续集成框架

 

Using curl POST with variables defined in bash script functions

https://stackoverflow.com/questions/17029902/using-curl-post-with-variables-defined-in-bash-script-functions