# -*- coding=utf-8 -*- #curl公共程序 pycurlcomm.py #张明伟 20200331 import pycurl from io import BytesIO import datetime,time,os import logging import json from logging import handlers testdebug=1 #测试模式：1为是，0为否 if testdebug: ip = '221.122.125.207' #测试环境IP else: ip = '221.122.125.201' #正式环境IP pth=os.getcwd()+os.sep+'data'+os.sep logpath =os.getcwd() + os.sep + 'logs' + os.sep #返回当前日期 def dtnow(): return datetime.datetime.today().strftime('%Y%m%d') #返回文件名相关信息 def filestr(filename): filepath,tmpfile=os.path.split(filename) basename,extention=os.path.splitext(tmpfile) return filepath,tmpfile,basename,extention #暂时不用 def init(): dt=(datetime.datetime.today()+datetime.timedelta(days=-1)).strftime('%Y-%m-%d') rdt=dt.replace('-','') logfile='bigdata'+rdt+'.log' logging.basicConfig(level=logging.INFO,format='%(asctime)s-%(filename)s.%(module)s.%(funcName)s:%(lineno)s:%(message)s',filename=logfile) logging.info("系统金额我发你") print(os.path.exists(logfile)) #curlcmd用于上传下载，scp文件，http请求 #op=download/upload 下载或上传 默认是下载，上传时需要传入filename,http请求时需要data def curlcmd(url,op="download",filename=None,data=""): if not filename: filename='' c = pycurl.Curl() c.setopt(pycurl.SSL_VERIFYPEER, False) #ssl不信任或无证书时需要设置这个 等同于curl -k/--insecure 参数 c.setopt(pycurl.SSL_VERIFYHOST, False) # c.setopt(pycurl.URL, url+filename) #上传时需要指定上传文件名，所以filename加入到url中 c.fp = BytesIO() #设置输出 c.setopt(pycurl.POST, 1) #以POST方式提交 # c.setopt(pycurl.POSTFIELDS, urllib.urlencode(data)) if isinstance(data,list): #提交大量数据 c.setopt(pycurl.HTTPPOST, data) elif isinstance(data,dict): #json格式的数据上传 对应着大量的表单数据提交 c.setopt(pycurl.POSTFIELDS,json.dumps(data)) c.setopt(pycurl.USERPWD, "ftpuser:Cbpp@123") #用户名密码 c.setopt(pycurl.WRITEFUNCTION, c.fp.write) #curl执行结果 #print(data,url,op,filename) if op=="upload": #上传文件 if not filename: raise Exception("请输入上传文件名!") filename1=pth+filename filename1size=os.path.getsize(filename1) c.setopt(pycurl.UPLOAD, True) #设置上传标志 c.setopt(pycurl.INFILESIZE, filename1size) #上传需要传入文件大小标志 # c.setopt(pycurl.READFUNCTION, open(filename1,"rd").read(filename1size)) c.setopt(pycurl.READFUNCTION, open(filename1, "rb").read) # print(data, '@@@@@@') # print(urllib.urlencode(data)) c.perform() #curl中真正的执行 code = c.getinfo(c.HTTP_CODE) #获取返回码 html = c.fp.getvalue() #获取返回结果 print(html) return {'code':code,'html':html} #返回json格式 #日志类 class Logger(object): level_relations={ 'debug':logging.DEBUG, 'info':logging.INFO, 'waining':logging.WARNING, 'error':logging.ERROR, 'crit':logging.CRITICAL } def __init__(self,filename=None,level='debug',when='D',backCount=3,fmt='%(asctime)s-%(filename)s.%(module)s.%(funcName)s:%(lineno)s:%(message)s'): if not os.path.exists(logpath): os.mkdir(logpath) if filename is None: filename=filestr(__file__)[2] #返回无后缀的文件名 logfile=logpath+filename+dtnow()+'.log' self.logger=logging.getLogger(logfile) format_str=logging.Formatter(fmt) self.logger.setLevel(self.level_relations.get(level)) sh=logging.StreamHandler() sh.setFormatter(format_str) th=handlers.TimedRotatingFileHandler(filename=logfile,when=when,backupCount=backCount) th.setFormatter(format_str) self.logger.addHandler(sh) self.logger.addHandler(th) #json数据类型解析 class expressDict(): fldlist = [] # type:list #字段名列表 valuelist = [] # type:list #值列表 sql = "" #SQL INSERT语句列表 valuesql = "" #SQL值语句列表 def __init__(self,jsstr,sqlquote="'"): self.jsstr = jsstr self.sqlquote = "'" self.tabname = "" self.sqls=[] self.sqlFlag = True #默认是需要得到SQL语句 # self.fldlist=[] #type:list # self.valuelist=[] #type:list # self.sql = "" # self.valuesql = "" #数据处理后再次重置数据 def __clear(self): self.fldlist = [] # type:list self.valuelist = [] # type:list self.sql = "" self.valuesql = "" def __display_item(self, jsstr): """ 根据json格式自动生成SQL语句或文件:递归 文件存放在{pth}目录下 sqlFlag默认为真，为真时产生sql语句，否则生成脚本 :param jsstr: :return: """ #文件格式存放时不需要分割符 if not self.sqlFlag: self.sqlquote="" #如果json格式中数据库字典格式，则把键值做为表名 if isinstance(jsstr, dict): for k,v in jsstr.items(): if not isinstance(v,list): self.fldlist.append(k) self.valuelist.append(v) # print(k,'--->',v) self.sql=self.sql+k+',' self.valuesql=self.valuesql+self.sqlquote+v+self.sqlquote+',' else: if not k: # print(self.tabname,"*" * 20) pass else: if k == self.tabname: self.sql = self.sql+') ' self.tabname=k # print(self.tabname+'@@@@@@@') self.__display_item(v) # 如果json格式中数据库列表格式，则把其他数据做库数据值 elif isinstance(jsstr, list): for jslst in jsstr: if self.sqlFlag: self.sql=self.sql+"insert into %s (" %(self.tabname) self.__display_item(jslst) self.sql = self.sql.strip(',') + ") values ("+self.valuesql.strip(',')+");" self.sqls.append(self.sql) self.__clear() else: finame = pth + self.tabname + '_' + dtnow() + '.del' self.__display_item(jslst) data = self.valuesql.strip(',') + os.linesep # type:str self.__clear() print(finame, data) with open(finame, 'a+') as fp: fp.write(data.encode("utf-8")) def display_items(self,sqlFlag=True): # return self.display_item(self.jsstr) self.sqlFlag=sqlFlag return self.__display_item(self.jsstr) def dispay_sqls(self): for sql in self.sqls: print(sql) # print(self.valuesql) # curl -k -v -T "{table_name1_20200403.del,table_name2_20200403.del}" -u ftpuser scp://221.122.2.196/tmp/ # print(curlcmd(data="",url="scp://221.122.2.196//tmp/gjj20200402.log")) # print(curlcmd(data="",url="scp://ftpdata@221.122.2.196/tmp/",op="upload",filename="test111.txt"))