zoukankan      html  css  js  c++  java
  • python-gitlab 一个简单demo

    背景

    • 需要收集git仓库信息到数据库供前端展示
      • 包括:仓库信息、仓库所有者、成员列表、提交信息、活跃情况等
    • 需要定时启动、灵活触发

    实现简介

    • 使用gitlab v4 restful 接口
    • 使用python-gitlab 依赖库(pyapi-gitlab 不支持gitlab 10.0+ 版本,故弃用)
    • 使用flask 提供接口,远程灵活触发
    • 使用requests 完成数据汇报与接口访问(因频率不高,性能优化相关暂不考虑)
    • 使用datetime/dateutil 完成gitlab 的时间信息向mysql datatime转化
    • 外层使用docker进行封装与部署

    api使用

    • projects
    
    
    class GitProjectEntry(CCInterfaceHandler):
        def __init__(self, _project):
            super(GitProjectEntry, self).__init__()
            self.logger = logger
            self.project = _project
            self.attribute = self.project.attributes
            self.logger.debug("parse project name : {PROJECT_NAME} with id : {PROJECT_ID}.".format(
                PROJECT_NAME = self._self_attribute_get("name"),
                PROJECT_ID = self.project.get_id()
            ))
    
            # id, name, description, owner, web_url, last_activity_at, created_at, num_commits, member_list
            self.KEY_ID = "id"
            self.KEY_NAME = "name"
            self.KEY_DESCRIPTION = "description"
    
            self.KEY_OWNER = "owner"
            self.KEY_OWNER_WEB_URL = "owner_web_url"
    
            self.KEY_LAST_ACTIVITY_AT = "last_activity_at"
            self.KEY_CREATED_AT = "created_at"
            self.KEY_ACTIVE_STATUS = "status"
    
            self.KEY_NAMESPACE_PATH = "namespace_path"
            self.KEY_NAMESPACE_WEB_URL = "namespace_web_url"
    
            self.KEY_MEMBER_USERNAME_LIST = "member_username_list"
    
            self.KEY_WEB_URL = "web_url"
            self.KEY_HTTP_URL_TO_REPO = "http_url_to_repo"
            self.logger.debug(str(self._self_init()))
    
        def _date_formatter(self, _git_date_stamp_str):
            try:
                return date_parser.parse(_git_date_stamp_str).strftime(database_datefmt)
            except Exception,e:
                self.logger.warn("date stamp convert failed. with {INNER_GIT_STAMP}".format(INNER_GIT_STAMP = _git_date_stamp_str))
    
        def _git_status_active_judge(self, _git_last_active_at_dt):
            # status 1 活跃,0不活跃
            KEY_STATUS_ACTIVE = 1
            KEY_STATUS_INACTIVE = 0
    
            today=datetime.datetime.today()
            month_time_delta=datetime.timedelta(days = 30, hours=1)
            activity_threshold = today - month_time_delta
            activity_status = KEY_STATUS_ACTIVE if(
                date_parser.parse(_git_last_active_at_dt) > activity_threshold) else KEY_STATUS_INACTIVE
            return activity_status
    
        def _self_init(self):
    
            self.member_list = []
            self.guess_owner = None
    
            self.last_activity_at = self._date_formatter(self._self_attribute_get("last_activity_at"))
    
            self.data = {
    
                self.KEY_ID  : self.project.get_id(), # self.project.id
                self.KEY_NAME : self._self_attribute_get("name"), # self.project.name
                self.KEY_DESCRIPTION : self._self_attribute_get("description"),
    
                self.KEY_OWNER : self._self_attribute_get("owner", {}).get("username"),
                self.KEY_OWNER_WEB_URL : self._self_attribute_get("owner", {}).get("web_url"),
    
                self.KEY_LAST_ACTIVITY_AT : self.last_activity_at,
                self.KEY_CREATED_AT : self._date_formatter(self._self_attribute_get("created_at")),
                self.KEY_ACTIVE_STATUS : self._git_status_active_judge(self.last_activity_at),
    
                self.KEY_NAMESPACE_PATH : self._self_attribute_get("namespace", {}).get("path"),
                self.KEY_NAMESPACE_WEB_URL : self._self_attribute_get("namespace", {}).get("web_url"),
    
                self.KEY_MEMBER_USERNAME_LIST : self._self_member_username_list(),
    
                self.KEY_WEB_URL : self._self_attribute_get("web_url"),
                self.KEY_HTTP_URL_TO_REPO : self._self_attribute_get("description")
    
            }
    
            if(self.data.get(self.KEY_OWNER) == None):
                self.data[self.KEY_OWNER] = self._self_member_top_access_user_name()
    
            if(self.data.get(self.KEY_OWNER_WEB_URL) == None):
                self.data[self.KEY_OWNER_WEB_URL] = self._self_member_top_access_user_name()
    
            self.git_project_member_report(self.project.get_id(),self.member_list)
            if('200' != str(self.git_project_report(self.data))):
                self.logger.warn("reported git_project failed! with project_id : {PR_ID} and project_name : {PR_NAME} .
    "
                                 "while data is {PR_DATA}".format(PR_ID = str(self.data.get(self.KEY_ID, "NULL_ID")),
                                                                  PR_NAME = str(self.data.get(self.KEY_NAME, "NULL_NAME")),
                                                                  PR_DATA = str(self.data)
                                                                  )
                                 )
            else:
                self.logger.info("reported git_project succeed! with project_id : {PR_ID} and project_name : {PR_NAME} .
    "
                                 "while data is {PR_DATA}".format(PR_ID = str(self.data.get(self.KEY_ID, "NULL_ID")),
                                                                  PR_NAME = str(self.data.get(self.KEY_NAME, "NULL_NAME")),
                                                                  PR_DATA = str(self.data)
                                                                  )
                                 )
            return self.data
    
        def _self_member_list(self):
            if(self.member_list != None and len(self.member_list) > 0):
                return self.member_list
            self.member_list =  self.project.members.list(all=True)
            if(len(self.member_list)<1):
                self.member_list = self.project.members.all(all=True)
            return self.member_list
    
        def _self_member_top_access_user(self):
            if(not self.guess_owner):
                self.member_list = self._self_member_list()
                self.sorted_member_list = sorted(self.member_list,key=lambda x : x.attributes.get("access_level",0),reverse=True)
                self.guess_owner = self.sorted_member_list[0] if(len(self.sorted_member_list) > 0) else None
            return self.guess_owner
    
        def _self_member_top_access_user_weburl(self):
            self.guess_owner = self._self_member_top_access_user()
            self.top_access_username = self.guess_owner.attributes.get("web_url", None) if(self.guess_owner) else None
            return self.top_access_username
    
        def _self_member_top_access_user_name(self):
            self.guess_owner = self._self_member_top_access_user()
            self.top_access_username = self.guess_owner.attributes.get("username", None) if(self.guess_owner) else None
            return self.top_access_username
    
        def _self_member_username_list(self):
            '''
            gitlab document:
                Examples
                List the project members:
    
                members = project.members.list()
                List the project members recursively (including inherited members through ancestor groups):
    
                members = project.members.all(all=True)
                Search project members matching a query string:
    
                members = project.members.list(query='bar')
                Get a single project member:
    
                member = project.members.get(user_id)
                Add a project member:
    
                member = project.members.create({'user_id': user.id, 'access_level':
                                                 gitlab.DEVELOPER_ACCESS})
                Modify a project member (change the access level):
    
                member.access_level = gitlab.MAINTAINER_ACCESS
                member.save()
                Remove a member from the project team:
    
                project.members.delete(user.id)
                # or
                member.delete()
                Share/unshare the project with a group:
    
                project.share(group.id, gitlab.DEVELOPER_ACCESS)
                project.unshare(group.id)
            :return:
            '''
            self.member_username_list = [
                member.attributes.get("username") for member in self._self_member_list()
            ]
            return self.member_username_list
    
        def _self_attribute_get(self, _key, _default_value = ""):
            return self.attribute.get(_key, _default_value)
    
        def to_json(self):
            self.json_string = json.dumps(self.data)
            return self.json_string
    
    
    • members
        def _self_member_username_list(self):
            '''
            gitlab document:
                Examples
                List the project members:
    
                members = project.members.list()
                List the project members recursively (including inherited members through ancestor groups):
    
                members = project.members.all(all=True)
                Search project members matching a query string:
    
                members = project.members.list(query='bar')
                Get a single project member:
    
                member = project.members.get(user_id)
                Add a project member:
    
                member = project.members.create({'user_id': user.id, 'access_level':
                                                 gitlab.DEVELOPER_ACCESS})
                Modify a project member (change the access level):
    
                member.access_level = gitlab.MAINTAINER_ACCESS
                member.save()
                Remove a member from the project team:
    
                project.members.delete(user.id)
                # or
                member.delete()
                Share/unshare the project with a group:
    
                project.share(group.id, gitlab.DEVELOPER_ACCESS)
                project.unshare(group.id)
            :return:
            '''
            self.member_username_list = [
                member.attributes.get("username") for member in self._self_member_list()
            ]
            return self.member_username_list
    
    
    • users
    In [260]: user.attributes
    Out[260]:
        {u'access_level': 50,
         u'avatar_url': None,
         u'expires_at': None,
         u'id': 224,
         u'name': u'mosaic',
         'project_id': 18915,
         u'state': u'active',
         u'username': u'mosaic',
         u'web_url': u'http://git.intra.mosaic.com/mosaic'}
    

    完整代码

    • 新增http连接池
    # !/usr/bin/python
    # -*- coding: UTF-8 -*-
    __author__ = 'enzhao'
    # Created by enzhao on 2020/5/25.
    
    # -*- coding: utf-8 -*-
    import gitlab
    from gitlab.v4 import objects as git_const
    import json
    import logging
    from logging.handlers import RotatingFileHandler
    # fh = RotatingFileHandler(filename, maxBytes=1024, backupCount=5)
    
    import datetime
    from dateutil import parser as date_parser
    import time
    from functools import wraps
    
    from flask import Flask
    import requests
    
    from gitlab.v4.objects import Project
    
    import sys
    reload(sys)
    sys.setdefaultencoding("utf-8")
    
    app = Flask(__name__)
    
    RESPONSE_SUCCESS_CODE = 200
    RESPONSE_FAILED_CODE = 500
    RESPONSE_SUCCESS_MESSAGE = "success"
    RESPONSE_RESULT_KEY = "result"
    RESPONSE_CODE_KEY = "code"
    RESPONSE_MESSAGE_KEY = "message"
    RESPONSE_PARAMS_KEY = "params"
    RESPONSE_DICT_TEMPLATE = {
        RESPONSE_RESULT_KEY  : {},
        RESPONSE_CODE_KEY    : RESPONSE_SUCCESS_CODE,
        RESPONSE_MESSAGE_KEY : RESPONSE_SUCCESS_MESSAGE,
        RESPONSE_PARAMS_KEY  : {}
    }
    
    logger_file_suffix = time.strftime('%Y-%m-%d', time.localtime( float( time.time() )))
    logger_file_name = "./waic-gitlab-info-retrive.log"
    logger_file_name = "".join(["./develop-python-gitlab/host-py-source/waic-gitlab-info-retrive-", logger_file_suffix, ".log"])
    # logger_file_name = "./waic-gitlab-info-retrive.log"
    
    logger_time_fmt = '%Y/%m/%d %H:%M:%S %A'
    datefmt='%Y-%m-%d %H:%M:%S@%A'
    database_datefmt = "%Y-%m-%d %H:%M:%S"
    git_datefmt = "%Y-%m-%d %H:%M:%S"
    
    CC_HOST = "http://10.73.13.163"
    # CC_HOST = "http://controlcenter.ds.sina.com.cn"
    
    def get_local_ip():
        local_ip = ''
        try:
            import socket
            socket_objs = [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]
            ip_from_ip_port = [(s.connect(('8.8.8.8', 53)), s.getsockname()[0], s.close()) for s in socket_objs][0][1]
            ip_from_host_name = [ip for ip in socket.gethostbyname_ex(socket.gethostname())[2] if not ip.startswith('127.')][:1]
            local_ip = [l for l in (ip_from_ip_port, ip_from_host_name) if l][0]
        except (Exception),e:
            print('get_local_ip found exception : %s' % e)
        return local_ip if('' != local_ip and None != local_ip) else socket.gethostbyname(socket.gethostname())
    
    
    def result_decorator(_func_obj):
        @wraps(_func_obj)
        def wrapTheFunction(*args, **kwargs):
            receive_time = time.time()
            # RESPONSE_DICT = RESPONSE_DICT_TEMPLATE.copy()
            RESPONSE_DICT = {
                RESPONSE_RESULT_KEY  : {},
                RESPONSE_CODE_KEY    : RESPONSE_SUCCESS_CODE,
                RESPONSE_MESSAGE_KEY : RESPONSE_SUCCESS_MESSAGE,
                RESPONSE_PARAMS_KEY  : {}
            }
            RESPONSE_DICT[RESPONSE_RESULT_KEY]["server_ip"] = get_local_ip()
            RESPONSE_DICT[RESPONSE_RESULT_KEY]["time"] = time.strftime(datefmt,time.localtime())
            result_dict = {}
            try:
                result_dict = _func_obj(*args, **kwargs)
            except Exception,e:
                finish_time = time.time()
                RESPONSE_DICT[RESPONSE_RESULT_KEY]["duration"] = str(finish_time - receive_time) + "ms"
                RESPONSE_DICT[RESPONSE_PARAMS_KEY].update(**kwargs)
                RESPONSE_DICT[RESPONSE_CODE_KEY] = RESPONSE_FAILED_CODE
                RESPONSE_DICT[RESPONSE_MESSAGE_KEY] = e.message
                return json.dumps(RESPONSE_DICT)
            RESPONSE_DICT[RESPONSE_RESULT_KEY].update(result_dict)
            RESPONSE_DICT[RESPONSE_PARAMS_KEY].update(**kwargs)
            finish_time = time.time()
            RESPONSE_DICT[RESPONSE_RESULT_KEY]["duration"] = str(finish_time - receive_time) + "ms"
            RESPONSE_DICT[RESPONSE_CODE_KEY] = RESPONSE_SUCCESS_CODE
            RESPONSE_DICT[RESPONSE_MESSAGE_KEY] = RESPONSE_SUCCESS_MESSAGE
            return json.dumps(RESPONSE_DICT)
        return wrapTheFunction
    
    
    def wei_logger(_cls_name = "logger_helper",
                   _log_level = logging.INFO):# !/usr/bin/python
        _format_str = '[%(levelname)s] : %(asctime)s %(filename)s[line:%(lineno)d]  %(message)s'
        logging.basicConfig(level=logging.DEBUG,
                            format=_format_str,
                            filename=logger_file_name,
                            datefmt=logger_time_fmt)
    
        console = logging.StreamHandler()
        console.setLevel(logging.INFO)
        console_formatter = logging.Formatter(_format_str)
        console.setFormatter(console_formatter)
    
        rotating_file_handler = RotatingFileHandler(logger_file_name, maxBytes=1024 * 1024 * 50, backupCount=5)
        rotating_file_handler.setLevel(logging.DEBUG)
        rotating_file_handler.setFormatter(console_formatter)
    
        logger = logging.getLogger(_cls_name)
        logger.addHandler(console)
        logger.addHandler(rotating_file_handler)
        logger.debug('%s logger init success!!' % _cls_name)
        return logger
    
    logger = wei_logger("GitManageEntry")
    
    def get_http_session(pool_connections=1, pool_maxsize=10, max_retries=3):
        '''
        http连接池
        pool_connections 要缓存的 urllib3 连接池的数量。
        pool_maxsize 要保存在池中的最大连接数。
        max_retries 每个连接的最大重试次数
        '''
        session =requests.session()
        # 创建适配器
        adapter = requests.adapters.HTTPAdapter(pool_connections=pool_connections,
                                                pool_maxsize=pool_maxsize, max_retries=max_retries)
        session.mount('http://', adapter)
        session.mount('https://', adapter)
        return session
    
    session = get_http_session()
    
    class CCInterfaceHandler(object):
        def __init__(self):
            self.cc_host = CC_HOST
            self.url_sep = "/"
            self.git_project_query_url = "waic/v1/data/git_project"
            self.git_project_add_url = "waic/v1/data/git_project/add"
            self.git_project_update_url = "waic/v1/data/git_project/update"
            self.git_project_member_query_url = "waic/v1/data/git_project_member"
            self.git_project_member_add_url = "waic/v1/data/git_project_member/add"
            self.git_project_member_update_url = "waic/v1/data/git_project_member/update"
            self.interface_git_project_query = self.url_sep.join([self.cc_host, self.git_project_query_url ])
            self.interface_git_project_add = self.url_sep.join([self.cc_host, self.git_project_add_url ])
            self.interface_git_project_update = self.url_sep.join([self.cc_host, self.git_project_update_url ])
            self.interface_git_project_member_query = self.url_sep.join([self.cc_host, self.git_project_member_query_url])
            self.interface_git_project_member_add = self.url_sep.join([self.cc_host, self.git_project_member_add_url ])
            self.interface_git_project_member_update = self.url_sep.join([self.cc_host, self.git_project_member_update_url ])
            self.response_format_param = {"responseFormat" : "json"}
            self.role_mapping = {
                git_const.ACCESS_GUEST : "guest",
                git_const.ACCESS_REPORTER : "reporter",
                git_const.ACCESS_DEVELOPER : "developer",
                git_const.ACCESS_MASTER : "master",
                git_const.ACCESS_OWNER : "owner",
            }
    
        def get_value(self, _key_path = "cluster.json.weibox_path", _dict = {}):
            if(_dict == None):
                return ""
            tmp_obj = _dict.get(_key_path)
            if(tmp_obj == None):
                point_idx = _key_path.find(".")
                if(point_idx < 0):
                    return ""
                tail_dict = _dict.get(_key_path[0:point_idx])
                if(isinstance(tail_dict,list)):
                    tail_dict = tail_dict[0] if len(tail_dict)>0 else None
                tmp_obj = self.get_value(_key_path[point_idx+1:], tail_dict)
            return tmp_obj
    
        def _git_project_report_interface(self, _id):
            self.cc_project_count = 0
            try:
                self.id_param = self.response_format_param.copy()
                self.id_param.update({"id" : _id})
                r = session.get(self.interface_git_project_query, params = self.id_param)
                self.cc_project_count = self.get_value("result.totalCount",r.json())
            except Exception,e:
                logger.warn(str(e) + e.message)
            return self.interface_git_project_update if(self.cc_project_count > 0) else self.interface_git_project_add
    
        def git_project_report(self, _project_data = {}):
            self.id = _project_data.get("id")
            try:
                self.interface_git_project_report = self._git_project_report_interface(_id=self.id)
                self.report_param = self.response_format_param.copy()
                self.report_param.update(_project_data)
                r = session.post(self.interface_git_project_report, self.report_param)
                return self.get_value("code",r.json())
            except Exception,e:
                logger.warn(str(e) + e.message)
    
        def _git_project_member_query(self, _id):
            try:
                self.id_param = self.response_format_param.copy()
                self.id_param.update({"id" : _id})
                r = session.get(self.interface_git_project_member_query,  params = self.id_param)
                self.cc_project_member = self.get_value("result.data",r.json())
                self.cc_project_member_username_set = set([
                    user_dict.get("user_id", "") for user_dict in self.cc_project_member
                ])
                return self.cc_project_member_username_set
            except Exception,e:
                logger.warn(str(e) + e.message)
    
        def _git_project_member_report_interface(self, _project_id, _user_id):
            self.cc_project_member_count = 0
            try:
                self.id_param = self.response_format_param.copy()
                self.id_param.update({"project_id" : _project_id, "user_id" : _user_id})
                r = session.get(self.interface_git_project_member_query, params = self.id_param)
                self.cc_project_count = self.get_value("result.totalCount",r.json())
            except Exception,e:
                logger.warn(str(e) + e.message)
            return self.interface_git_project_member_update if(self.cc_project_count > 0) else self.interface_git_project_member_add
    
        def git_project_member_report(self, _project_id, _project_member_data = []):
            self.id = _project_id
            try:
                for project_user in _project_member_data:
                    cc_project_user_role = self.role_mapping.get(project_user.attributes.get("access_level",git_const.ACCESS_DEVELOPER))
                    cc_project_user_id = project_user.attributes.get("username")
                    cc_project_member_id = "".join([str(_project_id), "0", str(project_user.attributes.get("id"))])
                    self.interface_git_project_member_report = self._git_project_member_report_interface(
                        _project_id=_project_id, _user_id=cc_project_user_id)
                    self.id_param = self.response_format_param.copy()
                    self.id_param.update({
                        "id" : cc_project_member_id,
                        "project_id" : _project_id,
                        "user_id" : cc_project_user_id,
                        "role" : cc_project_user_role
                    })
                    r = session.post(self.interface_git_project_member_report, self.id_param)
    
            except Exception,e:
                logger.warn(str(e) + e.message)
    
    
    
    
    class GitProjectEntry(CCInterfaceHandler):
        def __init__(self, _project):
            super(GitProjectEntry, self).__init__()
            self.logger = logger
            self.project = _project
            self.attribute = self.project.attributes
            self.logger.debug("parse project name : {PROJECT_NAME} with id : {PROJECT_ID}.".format(
                PROJECT_NAME = self._self_attribute_get("name"),
                PROJECT_ID = self.project.get_id()
            ))
    
            # id, name, description, owner, web_url, last_activity_at, created_at, num_commits, member_list
            self.KEY_ID = "id"
            self.KEY_NAME = "name"
            self.KEY_DESCRIPTION = "description"
    
            self.KEY_OWNER = "owner"
            self.KEY_OWNER_WEB_URL = "owner_web_url"
    
            self.KEY_LAST_ACTIVITY_AT = "last_activity_at"
            self.KEY_CREATED_AT = "created_at"
            self.KEY_ACTIVE_STATUS = "status"
    
            self.KEY_NAMESPACE_PATH = "namespace_path"
            self.KEY_NAMESPACE_WEB_URL = "namespace_web_url"
    
            self.KEY_MEMBER_USERNAME_LIST = "member_username_list"
    
            self.KEY_WEB_URL = "web_url"
            self.KEY_HTTP_URL_TO_REPO = "http_url_to_repo"
            self.logger.debug(str(self._self_init()))
    
        def _date_formatter(self, _git_date_stamp_str):
            try:
                return date_parser.parse(_git_date_stamp_str).strftime(database_datefmt)
            except Exception,e:
                self.logger.warn("date stamp convert failed. with {INNER_GIT_STAMP}".format(INNER_GIT_STAMP = _git_date_stamp_str))
    
        def _git_status_active_judge(self, _git_last_active_at_dt):
            # status 1 活跃,0不活跃
            KEY_STATUS_ACTIVE = 1
            KEY_STATUS_INACTIVE = 0
    
            today=datetime.datetime.today()
            month_time_delta=datetime.timedelta(days = 30, hours=1)
            activity_threshold = today - month_time_delta
            activity_status = KEY_STATUS_ACTIVE if(
                date_parser.parse(_git_last_active_at_dt) > activity_threshold) else KEY_STATUS_INACTIVE
            return activity_status
    
        def _self_init(self):
    
            self.member_list = []
            self.guess_owner = None
    
            self.last_activity_at = self._date_formatter(self._self_attribute_get("last_activity_at"))
    
            self.data = {
    
                self.KEY_ID  : self.project.get_id(), # self.project.id
                self.KEY_NAME : self._self_attribute_get("name"), # self.project.name
                self.KEY_DESCRIPTION : self._self_attribute_get("description"),
    
                self.KEY_OWNER : self._self_attribute_get("owner", {}).get("username"),
                self.KEY_OWNER_WEB_URL : self._self_attribute_get("owner", {}).get("web_url"),
    
                self.KEY_LAST_ACTIVITY_AT : self.last_activity_at,
                self.KEY_CREATED_AT : self._date_formatter(self._self_attribute_get("created_at")),
                self.KEY_ACTIVE_STATUS : self._git_status_active_judge(self.last_activity_at),
    
                self.KEY_NAMESPACE_PATH : self._self_attribute_get("namespace", {}).get("path"),
                self.KEY_NAMESPACE_WEB_URL : self._self_attribute_get("namespace", {}).get("web_url"),
    
                self.KEY_MEMBER_USERNAME_LIST : self._self_member_username_list(),
    
                self.KEY_WEB_URL : self._self_attribute_get("web_url"),
                self.KEY_HTTP_URL_TO_REPO : self._self_attribute_get("description")
    
            }
    
            if(self.data.get(self.KEY_OWNER) == None):
                self.data[self.KEY_OWNER] = self._self_member_top_access_user_name()
    
            if(self.data.get(self.KEY_OWNER_WEB_URL) == None):
                self.data[self.KEY_OWNER_WEB_URL] = self._self_member_top_access_user_name()
    
            self.git_project_member_report(self.project.get_id(),self.member_list)
            if('200' != str(self.git_project_report(self.data))):
                self.logger.warn("reported git_project failed! with project_id : {PR_ID} and project_name : {PR_NAME} .
    "
                                 "while data is {PR_DATA}".format(PR_ID = str(self.data.get(self.KEY_ID, "NULL_ID")),
                                                                  PR_NAME = str(self.data.get(self.KEY_NAME, "NULL_NAME")),
                                                                  PR_DATA = str(self.data)
                                                                  )
                                 )
            else:
                self.logger.info("reported git_project succeed! with project_id : {PR_ID} and project_name : {PR_NAME} .
    "
                                 "while data is {PR_DATA}".format(PR_ID = str(self.data.get(self.KEY_ID, "NULL_ID")),
                                                                  PR_NAME = str(self.data.get(self.KEY_NAME, "NULL_NAME")),
                                                                  PR_DATA = str(self.data)
                                                                  )
                                 )
            return self.data
    
        def _self_member_list(self):
            if(self.member_list != None and len(self.member_list) > 0):
                return self.member_list
            self.member_list =  self.project.members.list(all=True)
            if(len(self.member_list)<1):
                self.member_list = self.project.members.all(all=True)
            return self.member_list
    
        def _self_member_top_access_user(self):
            if(not self.guess_owner):
                self.member_list = self._self_member_list()
                self.sorted_member_list = sorted(self.member_list,key=lambda x : x.attributes.get("access_level",0),reverse=True)
                self.guess_owner = self.sorted_member_list[0] if(len(self.sorted_member_list) > 0) else None
            return self.guess_owner
    
        def _self_member_top_access_user_weburl(self):
            self.guess_owner = self._self_member_top_access_user()
            self.top_access_username = self.guess_owner.attributes.get("web_url", None) if(self.guess_owner) else None
            return self.top_access_username
    
        def _self_member_top_access_user_name(self):
            self.guess_owner = self._self_member_top_access_user()
            self.top_access_username = self.guess_owner.attributes.get("username", None) if(self.guess_owner) else None
            return self.top_access_username
    
        def _self_member_username_list(self):
            '''
            gitlab document:
                Examples
                List the project members:
    
                members = project.members.list()
                List the project members recursively (including inherited members through ancestor groups):
    
                members = project.members.all(all=True)
                Search project members matching a query string:
    
                members = project.members.list(query='bar')
                Get a single project member:
    
                member = project.members.get(user_id)
                Add a project member:
    
                member = project.members.create({'user_id': user.id, 'access_level':
                                                 gitlab.DEVELOPER_ACCESS})
                Modify a project member (change the access level):
    
                member.access_level = gitlab.MAINTAINER_ACCESS
                member.save()
                Remove a member from the project team:
    
                project.members.delete(user.id)
                # or
                member.delete()
                Share/unshare the project with a group:
    
                project.share(group.id, gitlab.DEVELOPER_ACCESS)
                project.unshare(group.id)
            :return:
            '''
            self.member_username_list = [
                member.attributes.get("username") for member in self._self_member_list()
            ]
            return self.member_username_list
    
        def _self_attribute_get(self, _key, _default_value = ""):
            return self.attribute.get(_key, _default_value)
    
        def to_json(self):
            self.json_string = json.dumps(self.data)
            return self.json_string
    
    class GitManageEntry(object):
        def __init__(self, _private_token, _fetch_all = False, _exclude_project = True, _visibility = git_const.VISIBILITY_PRIVATE):
            '''
            gitlab document:
                The API provides several filtering parameters for the listing methods:
    
                archived: if True only archived projects will be returned
                visibility: returns only projects with the specified visibility (can be public, internal or private)
                search: returns project matching the given pattern
                Results can also be sorted using the following parameters:
    
                order_by: sort using the given argument. Valid values are id, name, path, created_at, updated_at and last_activity_at. The default is to sort by created_at
                sort: sort order (asc or desc)
                # List all projects (default 20)
                projects = gl.projects.list(all=True)
                # Archived projects
                projects = gl.projects.list(archived=1)
                # Limit to projects with a defined visibility
                projects = gl.projects.list(visibility='public')
    
                # List owned projects
                projects = gl.projects.list(owned=True)
    
                # List starred projects
                projects = gl.projects.list(starred=True)
    
                # Search projects
                projects = gl.projects.list(search='keyword')
            :param _private_token:
            :param _fetch_all:
            :param _exclude_project:
            :param _visibility:
            '''
            self.url = "http://git.intra.weibo.com/"
            self.private_token = _private_token
            self.fetch_all = _fetch_all
            self.exclude_project = _exclude_project
            self.logger = logger
            self.visibility = _visibility
            self.git = gitlab.Gitlab(self.url, self.private_token)
            self.git.auth()
            self.username = self.git.user.username
            self.user_id = self.git.user.id
            self.logger_info = "parse private token for username : {USER_NAME} with id : {USER_ID}.".format(
                USER_NAME = self.username,
                USER_ID = self.user_id
            )
            print (self.logger_info)
            self.logger.info(self.logger_info)
            self.user_last_sign_in_at = self.git.user.last_sign_in_at
            self.user_last_activity_on = self.git.user.last_activity_on
    
            self.logger.info("start retrieve git projects")
            self.projects = self.git.projects.list(all=True, visibility = self.visibility) 
                if(self.fetch_all) else 
                self.git.projects.list(visibility = self.visibility)
            self.logger.info("retrieve git projects done!")
    
            self.gpe = [GitProjectEntry(_project=project) for project in self.projects]
            self.gpe_data = [_gpe.data for _gpe in self.gpe]
            self.data = {
                "url" : self.url ,
                "private_token" : self.private_token ,
                "username" : self.username ,
                "user_id" : self.user_id ,
                "user_last_sign_in_at" : self.user_last_sign_in_at ,
                "user_last_activity_on" : self.user_last_activity_on,
                "count_of_projects" : len(self.gpe)
            }
    
            if(not self.exclude_project):
                self.data.update({"projects" : self.gpe_data})
            self.logger.debug(str(self.data))
    
        def to_json(self):
            self.json_string = json.dumps(self.data)
            return self.json_string
    
    @app.route('/waic/git-retrieve/user-auth/<private_token>',methods = ['POST','GET'])
    @result_decorator
    def git_user_auth(private_token):
        gme = GitManageEntry(_private_token=private_token)
        return gme.data
        # return gme.to_json()
    
    @app.route('/waic/git-retrieve/preview/<private_token>',methods = ['POST','GET'])
    @result_decorator
    def git_retrieve_preview(private_token):
        gme = GitManageEntry(_private_token=private_token, _exclude_project=False)
        return gme.data
        # return gme.to_json()
    
    
    @app.route('/waic/git-retrieve/all/<private_token>',methods = ['POST','GET'])
    @result_decorator
    def git_retrieve_all_projects(private_token):
        gme = GitManageEntry(_private_token=private_token, _fetch_all=True,  _exclude_project=False)
        return gme.data
        # return gme.to_json()
    
    @app.route('/waic/git-retrieve/all-projects/<private_token>',methods = ['POST','GET'])
    @result_decorator
    def git_retrieve_all_internal_projects(private_token):
        gme = GitManageEntry(_private_token=private_token, _visibility=git_const.VISIBILITY_INTERNAL, _fetch_all=True,  _exclude_project=False)
        return gme.data
        # return gme.to_json()
    
    
    if __name__ == '__main__':
        app.run(host=get_local_ip())
        # job_state("12345678")
        token = "anNN8srBMewKzvokyxaL"
        # token = "yk-1v6J_txopP7x7xtif"
        # gme = GitManageEntry(_private_token=token, _fetch_all=True, _exclude_project=False)
        gme = GitManageEntry(_private_token=token, _exclude_project=False)
    
    
    code_stage = '''
    # -*- coding: utf-8 -*-
    import gitlab
    from gitlab.v4.objects import Project
    url = "http://git.intra.weibo.com/"
    token = "fv29Z9seyCs7xRhYSUt8"
    token = "anNN8srBMewKzvokyxaL"
    gl = gitlab.Gitlab(url, token, per_page=50)
    
    projects = gl.projects.list(all=True)
    
    class GitManageEntry(Project):
        def toJson(self):
    
    
            "url" : self.url ,
            "private_token" : self.private_token ,
            "git" : self.git ,
            "username" : self.username ,
            "user_id" : self.user_id ,
            "user_last_sign_in_at" : self.user_last_sign_in_at ,
            "user_last_activity_on" : self.user_last_activity_on ,
    
    
    # 登录
    gl = gitlab.Gitlab(url, token)
    
    # ---------------------------------------------------------------- #
    # 获取第一页project
    projects = gl.projects.list()
    # 获取所有的project
    projects = gl.projects.list(all=True)
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # 获取所有project的name,id
    for p in gl.projects.list(all=True, as_list=False):
        print(p.name, p.id)
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # 获取第一页project的name,id
    for p in gl.projects.list(page=1):
        print(p.name, p.id)
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # 通过指定id 获取 project 对象
    project = gl.projects.get(501)
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # 查找项目
    projects = gl.projects.list(search='keyword')
    # ---------------------------------------------------------------- #
    
    # ---------------------------------------------------------------- #
    # 创建一个项目
    project = gl.projects.create({'name':'project1'})
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # 获取公开的项目
    projects = gl.projects.list(visibility='public')  # public, internal or private
    # ---------------------------------------------------------------- #
    
    
    #  获取 project 对象是以下操作的基础
    
    
    # ---------------------------------------------------------------- #
    # 通过指定project对象获取该项目的所有分支
    branches = project.branches.list()
    print(branches)
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # 获取指定分支的属性
    branch = project.branches.get('master')
    print(branch)
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # 创建分支
    branch = project.branches.create({'branch_name': 'feature1',
                                      'ref': 'master'})
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # 删除分支
    project.branches.delete('feature1')
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # 分支保护/取消保护
    branch.protect()
    branch.unprotect()
    # ---------------------------------------------------------------- #
    
    
    
    
    
    # ---------------------------------------------------------------- #
    # 获取指定项目的所有tags
    tags = project.tags.list()
    
    # 获取某个指定tag 的信息
    tags = project.tags.list('1.0')
    
    # 创建一个tag
    tag = project.tags.create({'tag_name':'1.0', 'ref':'master'})
    
    # 设置tags 说明:
    tag.set_release_description('awesome v1.0 release')
    
    # 删除tags
    project.tags.delete('1.0')
    # or
    tag.delete()
    
    # ---------------------------------------------------------------- #
    # 获取所有commit info
    commits = project.commits.list()
    for c in commits:
        print(c.author_name, c.message, c.title)
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # 获取指定commit的info
    commit = project.commits.get('e3d5a71b')
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # 获取指定项目的所有merge request
    mrs = project.mergerequests.list()
    print(mrs)
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # 获取 指定mr info
    mr = project.mergesession.get(mr_id)
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    #  创建一个merge request
    mr = project.mergerequests.create({'source_branch':'cool_feature',
                                       'target_branch':'master',
                                       'title':'merge cool feature', })
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # 更新一个merge request 的描述
    mr.description = 'New description'
    mr.save()
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # 开关一个merge request  (close or reopen):
    mr.state_event = 'close'  # or 'reopen'
    mr.save()
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # Delete a MR:
    project.mergerequests.delete(mr_id)
    # or
    mr.delete()
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # Accept a MR:
    mr.merge()
    # ---------------------------------------------------------------- #
    
    
    # ---------------------------------------------------------------- #
    # 指定条件过滤 所有的merge request
    # state: state of the MR. It can be one of all, merged, opened or closed
    # order_by: sort by created_at or updated_at
    # sort: sort order (asc or desc)
    mrs = project.mergerequests.list(state='merged', sort='asc')  # all, merged, opened or closed
    # ---------------------------------------------------------------- #
    
    
    
    # ---------------------------------------------------------------- #
    # 创建一个commit
    data = {
        'branch_name': 'master',  # v3
        'commit_message': 'blah blah blah',
        'actions': [
            {
                'action': 'create',
                'file_path': 'blah',
                'content': 'blah'
            }
        ]
    }
    commit = project.commits.create(data)
    # ---------------------------------------------------------------- #
    
    
    
    # ---------------------------------------------------------------- #
    # Compare two branches, tags or commits:
    result = project.repository_compare('develop', 'feature-20180104')
    print(result)
    # get the commits
    
    for commit in result['commits']:
        print(commit)
    #
    # get the diffs
    for file_diff in result['diffs']:
        print(file_diff)
    # ---------------------------------------------------------------- #
    
    
    
    
    
    # ---------------------------------------------------------------- #
    # get the commits
    for commit in result['commits']:
        print(commit)
    #
    # get the diffs
    for file_diff in result['diffs']:
        print(file_diff)
    # ---------------------------------------------------------------- #
    
    
    
    
    '''
    
    
  • 相关阅读:
    HDU 1114 Piggy-Bank
    HDU 2955 Robberies
    NTOJ 290 动物统计(加强版)
    POJ 3624 Charm Bracelet
    HDU 2602 Bone Collector
    POJ 1523 SPF(无向图割顶)
    HDU 5311 Hidden String
    HDU 1421 搬寝室
    HDU 1058 Humble Numbers
    POJ 3259 Wormholes(spfa判负环)
  • 原文地址:https://www.cnblogs.com/suanec/p/13036291.html
Copyright © 2011-2022 走看看