Django实现OSS日志下载

Django实现OSS日志下载

一、python3.7安装

　　1、下载安装python3.7

[root@lvs-nginx1 ~]# wget https://www.python.org/ftp/python/3.7.3/Python-3.7.3.tgz
[root@lvs-nginx1 ~]# tar -xf Python-3.7.3.tgz
[root@lvs-nginx1 ~]# mkdir /usr/local/python3.7.3
[root@lvs-nginx1 ~]# cd Python-3.7.3
[root@lvs-nginx1 ~]# ./configure --prefix=/usr/local/python3.7.3 --enable-optimizations
[root@lvs-nginx1 ~]# make && make install
[root@lvs-nginx1 Python-3.7.3]# ln -s /usr/local/python3.7.3/bin/python3.7 /usr/bin/python3.7
[root@lvs-nginx1 Python-3.7.3]# python3.7 
Python 3.7.3 (default, Nov  7 2019, 16:07:52) 
[GCC 4.4.7 20120313 (Red Hat 4.4.7-23)] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> exit()

 二、创建Django

　　1、创建django

[root@k8s-es7 python]# /usr/local/python3.7/bin/django-admin startproject getoslogs
[root@k8s-es7 python]# cd getoslogs/
[root@k8s-es7 getoslogs]# python manage.py startapp getlogs			##创建app
[root@k8s-es7 getoslogs]# python manage.py migrate
Operations to perform:
  Apply all migrations: admin, auth, contenttypes, sessions
Running migrations:
  Applying contenttypes.0001_initial... OK
  Applying auth.0001_initial... OK
  Applying admin.0001_initial... OK
  Applying admin.0002_logentry_remove_auto_add... OK
  Applying admin.0003_logentry_add_action_flag_choices... OK
  Applying contenttypes.0002_remove_content_type_name... OK
  Applying auth.0002_alter_permission_name_max_length... OK
  Applying auth.0003_alter_user_email_max_length... OK
  Applying auth.0004_alter_user_username_opts... OK
  Applying auth.0005_alter_user_last_login_null... OK
  Applying auth.0006_require_contenttypes_0002... OK
  Applying auth.0007_alter_validators_add_error_messages... OK
  Applying auth.0008_alter_user_username_max_length... OK
  Applying auth.0009_alter_user_last_name_max_length... OK
  Applying auth.0010_alter_group_name_max_length... OK
  Applying auth.0011_update_proxy_permissions... OK
  Applying sessions.0001_initial... OK
[root@k8s-es7 getoslogs]# python manage.py createsuperuser
Username (leave blank to use 'root'): root
Email address: 547253687@qq.com
Password: 
Password (again): 
This password is too short. It must contain at least 8 characters.
This password is too common.
This password is entirely numeric.
Bypass password validation and create user anyway? [y/N]: y
Superuser created successfully.

 　　此时在getoslogs/settings.py里面INSTALLED_APPS添加刚刚创建的getlogs，将ALLOWED_HOSTS替换成ALLOWED_HOSTS = ['*']，允许所有人访问，这时候就创建好了项目和app。启动

　　python manget.py runserver 0.0.0.0:5566，这时候django已经正常启动，访问如下图

　　2、添加py脚本

　　这个脚本很简单，就是两个函数，listfile函数用来获取前端输入的时间，oss api来获取到该时间下的备份日志文件，返回给前端。

　　filejiedong这个函数用来获取前段勾选的文件，解冻以及下载文件

[root@k8s-es7 getoslogs]# cd /home/pb/python/test/
[root@k8s-es7 test]# cat listdir.py 
#!/usr/local/bin/python3.7
# -*- coding: utf-8 -*-

import oss2,time,sys;
from flask import Flask,send_file
from django.shortcuts import render
from django.http import HttpResponse;
sys.path.append('/home/pb/python/getoslogs/getlogs');
import views;
import threading;

def environment(aliyun):
    if aliyun == "jinrongyun":
        auth = oss2.Auth('yourAccessKeyId','yourAccessKeySecret');
        bucket = oss2.Bucket(auth, 'http://oss-cn-hangzhou.aliyuncs.com', 'yourBucketName');
        return bucket;
    else:
        auth = oss2.Auth('yourAccessKeyId','yourAccessKeySecret');
        bucket = oss2.Bucket(auth, 'http://oss-cn-hangzhou.aliyuncs.com', 'yourBucketName');
        return bucket;

def listfile(date_time,bucket):
    data = [];
    data1 = [];
    for obj in oss2.ObjectIterator(bucket, prefix = date_time):
        data.append(obj.key)
    for x in range(len(data)):
        if "tar.gz" in data[x]:
            data1.append(data[x])
    return data1;

def filejiedong(objectname,bucket,cc):
    #cc = [];
    file_list = [];
    file_list.append(objectname);
    meta = bucket.head_object(objectname);
    if "expiry-date" not in str(meta.resp.headers):
        bucket.restore_object(objectname);
        while True:
            time.sleep(3);
            meta = bucket.head_object(objectname);
            if "expiry-date"  in str(meta.resp.headers):
                requesturl=(bucket.sign_url('GET',objectname,36000));
                cc.append(requesturl);
                return cc;
                break;
            else:
                continue;
    else:
        requesturl=(bucket.sign_url('GET',objectname,36000));
        cc.append(requesturl);
        return cc;

def main(objectname,bucket):
    #存放线程的list
    cc = [];
    threads = [];
    for file_name in objectname:
        print("filename:",file_name);
        t = threading.Thread(target=filejiedong,args=(file_name,bucket,cc,));
        threads.append(t);

    for t in threads:
        t.start();
        #join方法作用是，让主线程等待该线程结束，所以join应该放在下面的循环里，放在这里就会导致这个启动线程的循环阻塞。结果就是顺序执行。
        #join也可以设置超时时间
        #join函数只在主线程进行等待的时候有用，如果主线程还有其他事情，没必要调用join阻塞自己。
        #t.join()
    for t in threads:
        t.join();
    return cc;

　　然后去前端页面准备一个简单的html页面　

[root@k8s-es7 getlogs]# mkdir /home/pb/python/getoslogs/getlogs/templates
[root@k8s-es7 getlogs]# cd /home/pb/python/getoslogs/getlogs/templates
[root@k8s-es7 getlogs]# cat templates/listfile.html 
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>日志下载</title>
<link rel="stylesheet" href="../../static/css/layui.css" media="all">
</head>
<body>
    <h1>日志下载</h1>
    <h2>请根据实际情况填写以下内容</h2>
    <form method="post" action="/listfile/">
            <!-- <input type="text" name="file_date" placeholder="日期"> -->
            <div>
                <label><input type="radio" name="aliyun" value="jinrongyun">金融云</label>
                <label><input type="radio" name="aliyun" value="gongyouyun">公有云</label>
            </div>
            <div class="layui-inline">
                <!-- 注意：这一层元素并不是必须的 -->
                <input type="text" class="layui-input" id="test1" name="file_date"
                    placeholder="yyyy-MM-dd">
            </div>
            <br> {{ error }}
            <br>
            <button id="btn" type="submit" class="layui-btn layui-btn-sm">查询</button>
            {% csrf_token %}
        
    </form>
    <form method="post" action="/listfile/">
        <button id="btn" type="submit"
            class="layui-btn layui-btn-normal layui-btn-sm">解冻</button>
                <input type="text" value="{{ aliyun }}" name="aliyun">
        {% for url in result %}
        </br>
            <input type="checkbox" value="{{ url }}" name="file_jiedong">
            <a> {{url}} </a>
            {% endfor %}
        {{ error }}<br>
        {% csrf_token %}

    </form>



        {% for url in result1 %}
        </br>
        <a href={{url}}>{{url}}</a>
            {% endfor %}
        {{ error }}<br>
        {% csrf_token %}


    <script src="../../static/layui.js"></script>
    <script>
        layui.use('laydate', function() {
            var laydate = layui.laydate;

            //执行一个laydate实例
            laydate.render({
                elem : '#test1' //指定元素
            });
        });
    </script>
</body>
</html>

　　添加一个login界面，可以自己去网上找模板下载

[root@k8s-es7 getlogs]# cat templates/index.html 
<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="utf-8">
  <title>HTML5响应式用户登录界面模板</title>
  <meta name="description" content="particles.js is a lightweight JavaScript library for creating particles.">
  <meta name="author" content="Vincent Garreau" />
  <meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, user-scalable=no">
  <link rel="stylesheet" media="screen" href="/static/css/style.css">
  <link rel="stylesheet" type="text/css" href="/static/css/reset.css"/>
</head>
<body>

<form method="post" action="/login/">
<div id="particles-js">
		<div class="login">
			<div class="login-top">
				登录
			</div>
			<div class="login-center clearfix">
				<div class="login-center-img"><img src="/static/img/name.png"/></div>
				<div class="login-center-input">
					<input type="text" name="username" value="" placeholder="请输入您的用户名" onfocus="this.placeholder=''" onblur="this.placeholder='请输入您的用户名'"/>
					<div class="login-center-input-text">用户名</div>
				</div>
			</div>
			<br> {{ error }}
			<br>
			<div class="login-center clearfix">
				<div class="login-center-img"><img src="/static/img/password.png"/></div>
				<div class="login-center-input">
					<input type="password" name="password" value="" placeholder="请输入您的密码" onfocus="this.placeholder=''" onblur="this.placeholder='请输入您的密码'"/>
					<div class="login-center-input-text">密码</div>
				</div>
			</div>
			<!-- <div type="submit" class="login-button">
				登录
			</div> -->
			<button type="submit" class="login-button">
				登录
			</button> 
			{% csrf_token %}
		</div>
		<div class="sk-rotating-plane"></div>
</div>
</form>

<!-- scripts -->
<script src="/static/js/particles.min.js"></script>
<script src="/static/js/app.js"></script>
<script type="text/javascript">
	function hasClass(elem, cls) {
	  cls = cls || '';
	  if (cls.replace(/s/g, '').length == 0) return false; //当cls没有参数时，返回false
	  return new RegExp(' ' + cls + ' ').test(' ' + elem.className + ' ');
	}
	 
	function addClass(ele, cls) {
	  if (!hasClass(ele, cls)) {
	    ele.className = ele.className == '' ? cls : ele.className + ' ' + cls;
	  }
	}
	 
	function removeClass(ele, cls) {
	  if (hasClass(ele, cls)) {
	    var newClass = ' ' + ele.className.replace(/[	
]/g, '') + ' ';
	    while (newClass.indexOf(' ' + cls + ' ') >= 0) {
	      newClass = newClass.replace(' ' + cls + ' ', ' ');
	    }
	    ele.className = newClass.replace(/^s+|s+$/g, '');
	  }
	}
		document.querySelector(".login-button").onclick = function(){
				addClass(document.querySelector(".login"), "active")
				setTimeout(function(){
					addClass(document.querySelector(".sk-rotating-plane"), "active")
					document.querySelector(".login").style.display = "none"
				},800)
				setTimeout(function(){
					removeClass(document.querySelector(".login"), "active")
					removeClass(document.querySelector(".sk-rotating-plane"), "active")
					document.querySelector(".login").style.display = "block"
					alert("登录成功")
					
				},5000)
		}
</script>
<div style="text-align:center;">
<p>更多模板：<a href="http://www.mycodes.net/" target="_blank">源码之家</a></p>
</div>
</body>
</html>

　　有了页面之后，就需要创建一个url指向这个这个页面，修改编写view.py里面的listfile函数

[root@k8s-es7 getlogs]# cat /home/pb/python/getoslogs/getoslogs/urls.py 
"""getoslogs URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from getlogs import views            #将createyaml这个app的views变量插进来

urlpatterns = [
    path('admin/', admin.site.urls),
    path(r'listfile/', views.listfile),            #将这个listfile url指向getlogs/views.py里面的listfile函数
    path(r'login/', views.login),                #登录页面
]        

 　　 提前安装好oss2模块 pip install aliyun-python-sdk-core-v3　　

[root@k8s-es7 getlogs]# cat /home/pb/python/getoslogs/getlogs/views.py 
[root@k8s-es7 getlogs]# cat views.py 
# -*- coding:utf-8 -*-
from django.shortcuts import render
from django.http import HttpResponse;
import subprocess;
import os,oss2,ldap3;
import sys;
import json;
from django.shortcuts import render_to_response
from django.template import RequestContext


sys.path.append('/home/pb/python/test');
import listdir;
from ldap import LdapAdmin;

##查询某个日期下oss备份的日志
def listfile(request):
    if request.method == 'POST':
        file_date = request.POST.get('file_date', '')
        file_jiedong = request.POST.getlist('file_jiedong', '')
        aliyun = request.POST.get('aliyun', '');
        bucket = listdir.environment(aliyun); 
        env = request.POST.get('env')
        if file_date:
            result = listdir.listfile(file_date.strip(),bucket);
            context = {'result': result, 'aliyun': aliyun}
            return render(request,'listfile.html', context)
        else:
            pass;
        if file_jiedong:
            aliyun = request.POST.get('aliyun', '');
            bucket = listdir.environment(aliyun);
            #context = for_file(file_jiedong,bucket);
            context = {'result1': listdir.main(file_jiedong,bucket)};
            print("context",context);
            return render(request,'listfile.html', context)
        else:
            pass;
    else:
        return render(request,'listfile.html');

#def for_file(file_jiedong,bucket):
#    result_file = []
#    for i in file_jiedong:
#        #result1 = listdir.filejiedong((i).strip(),bucket);
#        result1 = listdir.main((i).strip(),bucket);
#        print("result1:",result1);
#        result_file.append(result1)
#        context = {'result1': result_file}
#    return context;

##登录脚本
def login(request):
    if request.method == 'POST':
        username = request.POST.get('username', '');
        password = request.POST.get('password', '');
        #if username:
        #    print("username: {}".format(username));
        #    #return render(request,'index.html', context)
        #else:
        #    pass;
        #if password:
        #    print("password: {}".format(password));
        #    #return render(request,'index.html', context)
        #else:
        #    pass;
        ldap_ins = LdapAdmin();
        resp = ldap_ins.login(username, password);
        if resp == True:
            return render(request,'listfile.html');
        else:
            return render(request,'index.html');
    else:
        return render(request,'index.html')
#def login_ldap(username):
#    if request.method == 'POST':
#        username = request.POST.get('username', '');
#        #password = request.POST.get('password', '');
#        print("username: ".format(username));
#        #print("password: ".format(password));
#    else:
#        return render(request,'index.html')


#def add_dict(list_test):
#    context = {}
#    a = 0
#    for i in list_test:
#        a = a + 1
#        context["result"+str(a)] = i
#    return context

　　

　　添加ldap脚本

[root@k8s-es7 getlogs]# cat /home/pb/python/test/ldap.py 
#! /usr/bin/python
# -*- coding:utf-8 -*-
# Author: panb

import logging
from ldap3 import Server, Connection, ALL

logger = logging.getLogger("oauth")

LDAP = {
    "server": "172.27.27.220",
    "port": 389,
    "use_ssl": False,
    "domain": "jcici.com",
    "base": "ou=People,dc=jcici,dc=com"
}


class LdapAdmin(object):
    def __init__(self):
        """
        init
        """
        self.host = LDAP['server']
        self.port = LDAP.get('port', 389)
        self.use_ssl = LDAP.get('use_ssl', False)
        self.domain = LDAP['domain']
        self.base = LDAP['base']
        self.search_filter = "uid={uid}"

    def login(self, username, password):
        """
        登录
        :return:
        """
        server = Server(host=self.host,
                        port=self.port,
                        use_ssl=self.use_ssl,
                        connect_timeout=15,
                        get_info=ALL)

        try:
            conn = Connection(server,
                              user=f"uid={username},{self.base}",
                              password=password,
                              check_names=True,
                              lazy=False,
                              auto_bind=True,
                              receive_timeout=30
                              )

        except Exception as e:
            err_msg = f'LDAP 认证失败:{e}'
            logger.error(err_msg)
            return False
        else:
            msg = conn.result
            #print(msg)
            conn.unbind()
            return True

        # print(server.info)
        # print(server.schema)
        # _username = (conn.extend.standard.who_am_i())
        # print(_username)


#ldap_ins = LdapAdmin()
#resp = ldap_ins.login(username, password) 
#print(resp)

 　　还需要在getoslogs/settings.py添加static

STATIC_URL = '/static/'
HERE = os.path.dirname(os.path.abspath(__file__))
HERE = os.path.join(HERE, '../')
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don‘t forget to use absolute paths, not relative paths.
    os.path.join(HERE, 'static/'),
)

 　　3、最终访问效果