zoukankan      html  css  js  c++  java

  • 常用命令.md

    一、常用命令

    1. #docker启动为知笔记 
    2. sudo docker run --restart=on-failure:3 --name wiz -it -d -v /volume2/docker/wiz/data:/wiz/storage -v /etc/localtime:/etc/localtime -p 8888:80 -e SEARCH=true wiznote/wizserver 
    3. sudo docker exec -it 775c7c9ee1e1 /bin/bash #进入容器 
    4. docker logs -f -t --since="2017-05-31" --tail=10 edu_web_1 #--since : 此参数指定了输出日志开始日期，即只输出指定日期之后的日志。-f : 查看实时日志-t : 查看日志产生的日期-tail=10 : 查看最后的10条日志。edu_web_1 : 容器名称 
    5. cd /var/lib/docker/containers/容器日志 
    6. find -name '*zookeeper*' 
    7. ps -ef|grep zookeeper --color 
    8. # 从主机复制到容器sudo docker cp host_path containerID:container_path 
    9. #从容器复制到主机sudo docker cp containerID:container_path host_path 
    10. #使用Dockerfile创建新的镜像(注意：末尾有一个点，表示当前路径):docker build -t "repository/images_name:tag" .  
    11. docker commit 80daedc72971 docker.io/jiama/kylinchange:v2.1 
    12. docker tag kylinchange docker.io/jiama/kylinchange:v2.1 
    13. docker push docker.io/jiama/kylinchange:v2.1 
    14. docker save -o xxx.tar.gz docker.io/jiama/kylinchange:v4.0 
    15. #shell脚本 
    16. #查找目录中的hadoop包含字符串内容那个的文件： 
    17. for file in `hadoop fs -ls /user/epg/2020/1/11 |awk '{print $NF}'`; do echo $file; hadoop fs -text $file |grep "e_type':'open_advert.*p_type.*fa:16:3e:85:61:35" --color ; done 
    18. hadoop fs -text /user/epg/2020/02/08/epg_log_2020020800????.log|grep "e_type':'order.*c8:1f:66:df:de:f0"|wc -l 
    19. #hdfs清空回收站 
    20. hdfs dfs -expunge  
    21. #日志切割命令 
    22. split -b 10m catalina.out 切割后的日志名前缀 
    23. #批量删除后缀 
    24. for i in `ls epglogs_send |grep epg-access-2019-08-13*`;do cp $i epglog20190914/${i%.*};done 
    25. #存在的问题解决 
    26. #cp: cannot stat ‘epg-access-2019-08-13-23-48.log.COMPLETED’: No such file or directory 
    27. chown -R root:root gansulogs/ 
    28. chmod -R 666 gansulogs/ 
    29. mkdir epglog20190914 
    30. #注意cp命令后的文件路径 
    31. for i in `ls gansulogs | grep epg-access*`;do cp gansulogs/$i gansulogs/${i%.*};done 
    32. #虚拟机 
    33. #启动kafka 
    34. nohup /opt/kafka_2.11-0.10.2.0/bin/kafka-server-start.sh /opt/kafka_2.11-0.10.2.0/config/server.properties > /opt/scriptlogs/kafka.log 2>&1 
    35. #启动flume 
    36. nohup /opt/flume/bin/flume-ng agent --conf conf --conf-file /opt/pukkasoft/flume/conf/nginx_kafka.properties --name myAgent -Dflume.root.logger=INFO,console > /opt/logs/flume.log 2>&1 
    37. #创建消费者 
    38. /opt/kafka_2.11-0.10.2.0/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic ipvtepglogs 
    39. #查看kafka leader 
    40. /opt/kafka_2.11-0.10.2.0/bin/kafka-topics.sh --describe --topic report-request --zookeeper 192.168.20.160:2182,192.168.31.6:2181/apachekfk,192.168.31.7:2181/apachekfk 
    41. #启动spark 
    42. /opt/spark-1.6.3-bin-hadoop2.6/sbin/start-all.sh  
    43.  
    44. kafka-topics --delete --zookeeper 192.168.6.111:2181 --topic iptvepglogs #过一会主题就会被删掉 
    45.  
    46. 连接hive 
    47. beeline -u jdbc:hive2://sh-dev3-demo:10000 -n hdfs 
    48.  
    49. #启动redis 
    50. redis-server /opt/redis-3.2.8/redis.conf 
    51. #CDH启动命令 
    52. /opt/cloudera-manager/cm-5.16.1/etc/init.d/cloudera-scm-server start 
    53. /opt/cloudera-manager/cm-5.16.1/etc/init.d/cloudera-scm-agent start 
    54. service restart cloudera-scm-server  
    55. #cdh组件正在使用的配置 
    56. /var/run/cloudera-scm-agent/process  
    57. #广信 
    58. #广信回传ftp 
    59. nohup sh -x /home/pake/scripts/log_ftp_2.sh 2019-08-16 2019-08-20 > /home/pake/logs/test.log 2>&1 & 
    60.  
    61. #mysql后台脚本执行脚本 
    62. nohup mysql -h 192.168.10.45 -uroot -ppukkasoft2016 -Dintelligentrecommend < /home/pake/deleteWhere.sql > /home/pake/logs/delete.log 2>&1 & 
    63.  
    64. #ERROR 1205 (HY000) at line 1: Lock wait timeout exceeded; try restarting transaction 
    65.  
    66. #数据库操作最好分批进行特别是删除操作! 
    67. delete from intelligentrecommend.user_portrait_or_label_statistics where id>1400000 and id<'1600000' and content_kind_name ='精神'; 
    68. 创建数据库： 
    69. CREATE DATABASE IF NOT EXISTS cmserver default character set utf8 COLLATE utf8_general_ci; 
    70. grant all privileges on sentry.* to 'sentryuser'@'%' identified by 'sentry321' with grant option; 
    71.  
    72. #根据标签查询推荐内容 
    73. select t1.contentcode as code,t1.contentname as name,'' as PosterImg,contenttype from t_cp_tag_content t1 where t1.TagName='文艺' and t1.ContentType in( 'program','series' ) and t1.ContentID not in (select t2.ContentID from t_cp_tag_content t2 where t2.TagName='不可推荐') limit 3; 
    74.  
    75. #快捷键 
    76. #idea 
    77. ctrl+r快速替换 
    78. ctrl+shift+f全工程搜索 
    79.  
    80. #广信实时推荐job 
    81. nohup spark-submit --name "报表统计" --class cn.pukkasoft.rec.job.CommonStatistics --master spark://192.168.31.5:7077 --deploy-mode cluster hdfs://bigdata-cdh001.pukka.cn:8020/jobs/rec-statistics-job-20190908.jar jdbc:mysql://192.168.31.5:3306/ottdbsxxy?characterEncoding=utf8&useSSL=false root root hdfs://192.168.31.5:8020:/home/data/epg/2019/09/07/ 0931 > /opt/scriptlogs/testjob.log 2>&1 
    82.  
    83. #查看yarn程序日志 
    84. yarn logs -applicationId  
    85.  
    86. 测试日志写kafka 
    87. #!/bin/bash 
    88. i=1; 
    89. while  
    90. (( i <= $1 )) 
    91. do 
    92. echo ["/log/s.gif?d={'c_content_id':1775291,'c_content_code':'cce07ad7a80a49a883f0579023c147ed','c_content_name':'精灵梦叶罗丽第 
    93. 七季 第1集','c_series_id':1775289,'c_series_code':'BC2C5B4000AF4D00FECF128A445D1654','c_series_name':'[HD]精灵梦叶罗丽 第七季','c_content_type':'series','c_sequence':1,'py_duration':84420,'py_times':0,'c_actors':'未知','c_writers':'未知','c_release_year':2016,'c_category_id':'1430,1439,6565,6595','c_category_name':'最新热播,全部少儿,7-12岁,魔法少女集结 闪耀吧仙子们','c_content_kind_id':'1629','c_content_kind_name':'少儿','c_cp_id':'11','c_cp_name':'MGTV_infuseCP','c_sp_id':11,'c_sp_name':'华为(青海)','o_id':'8001013020','o_name':'芒果包月 
    94. 包','c_is_pay':1,'u_id':'09768871181itv','u_group':'10','u_epg_group':'defaultnew','u_stbid':'00100516060100A000007847E31442DE','p_mark':'mango_play','e_type':'play','e_time':1567859185521,'p_action_time':1020087,'p_url':'http://125.72.108.136:8081/EPG720P/page/play.html?returnUrl=http%3A%2F%2F125.72.108.136%3A8081%2FEPG720P%2Fpage%2Fdetail-num.html%3FcontentId%3D1775289%26playNum%3D1%26recommendCode%3D1430','p_referrer':'http://125.72.108.136:8081/EPG720P/page/detail-num.html?contentId=1775289&contentType=3&recommendCode=1430','es_template_id':'0','es_site_name':'mango','p_type':'play','p_title':'芒果专区播放页','py_action':'heartbeat'}","07/Sep/2019:20:26:24 +0800","90:B1:1C:33:A0:44","192.168.11.27"]; 
    95. let i++; 
    96. done | kafka-console-producer --broker-list bdp-reckon01-pukka:9092,bdp-reckon03-pukka:9092,bdp-reckon04-pukka:9092 --topic iptvepglogs 
    97.  
    98. sleep 10 
    99. exit 0 
    100.  
    101. #tomcat启动关闭 
    102. /opt/context/tomcat7094/bin/startup.sh  
    103. /opt/context/tomcat7094/bin/shutdown.sh 
    104.  
    105.  
    106. #nodejs 启动命令 
    107. ./restart.sh prod 
    108. 前臺運行npm run start 
    109.  
    110. #清空数据库 
    111. ##查出所有trunate语句 
    112. select CONCAT('truncate TABLE ',table_schema,'.',TABLE_NAME, ';') from INFORMATION_SCHEMA.TABLES where table_schema in ('iptvbigdata'); 
    113. ##复制查到的结果粘贴到查询中执行即可 
    114.  
    115. #杀死筛选进程号 
    116. kill `ps -ef|grep spark|awk '{print $2}'` 
    117.  
    118. #unzip命令解压缩jar包 
    119. unzip xxx-web-role-1.0-SNAPSHOT.jar -d /APP/www/idoxu 
    120. jar -cvfM0 project.jar ./ 
    121.  
    122.  
    123. ## kerbors 
    124. ##登录 
    125. kinit -kt /run/cloudera-scm-agent/process/873-hdfs-NAMENODE-nnRpcWait/hdfs.keytab hdfs/bdp-reckon01-pukka@bdp-reckon01-pukka 
    126.  
    127. #presto启动所有节点 
    128. /opt/cloudera/parcels/presto/bin/launcher start 
    129. 界面http://192.168.6.124:9999/ui/ 
    130.  
    131. #hivesql 
    132. #增加列 
    133. alter table dwd_play_fact_di add columns(`c_series_id` string COMMENT '剧头id',`c_series_name` string COMMENT '剧头名称',`c_series_code` string COMMENT '剧头code') 
    134.  
    135. #获取完整的oozie作业日志 
    136. oozie job -log <job_id> 
    137.  
    138. #hive删除分区 
    139. ALTER TABLE dwd_order_fact_di DROP IF EXISTS PARTITION (dt='2020-01-04');##测试没有成功删除 

    二、hql总结核心业务与关键字

    1. 累计: sum(xx) group by
    2. topN： row_number() over(partition by xx order by yy desc) rank
    3. a表存在b表不存在： left join on a.xx=b.xx and|where
      join on and 不会过滤结果记录条数，只会根据and后的条件是否显示 B表的记录，A表的记录一定会显示
    • on and条件是在生成临时表时使用的条件，它不管on中的条件是否为真，都会返回左边表中的记录。
    • where条件是在临时表生成好后，再对临时表进行过滤的条件。这时已经没有left join的含义（必须返回左边表的记录）了，条件不为真的就全部过滤掉。
    1. 合并a表和b表数据: union [all] 所以加all更快
    • 对重复结果的处理：UNION在进行表链接后会筛选掉重复的记录，Union All不会去除重复记录。
    • 对排序的处理：Union将会按照字段的顺序进行排序；UNION ALL只是简单的将两个结果合并后就返回
    1. 行专列，列转行=>核心是把key的列枚举分类统计查询出value值的聚合结果
    1. insert into TEST_TB_GRADE(USER_NAME, COURSE, SCORE) values 
    2. ("张三", "数学", 34), 
    3. ("张三", "语文", 58), 
    4. ("张三", "英语", 58), 
    5. ("李四", "数学", 45), 
    6. ("李四", "语文", 87), 
    7. ("李四", "英语", 45), 
    8. ("王五", "数学", 76), 
    9. ("王五", "语文", 34), 
    10. ("王五", "英语", 89); 
    11. ---行专列 
    12. SELECT user_name , 
    13. MAX(CASE course WHEN '数学' THEN score ELSE 0 END ) 数学, 
    14. MAX(CASE course WHEN '语文' THEN score ELSE 0 END ) 语文, 
    15. MAX(CASE course WHEN '英语' THEN score ELSE 0 END ) 英语 
    16. FROM test_tb_grade 
    17. GROUP BY USER_NAME; 
    18. ---列转行 
    19. select user_name, '语文' COURSE , CN_SCORE as SCORE from test_tb_grade2 
    20. union select user_name, '数学' COURSE, MATH_SCORE as SCORE from test_tb_grade2 
    21. union select user_name, '英语' COURSE, EN_SCORE as SCORE from test_tb_grade2 
    22. order by user_name,COURSE; 

    money怎么来？
  • 相关阅读:
    JSON基础(Java)
    美式英语音标词对照表
    network adapter
    debian网络静态ip配置
    apt --fix-broken install
    CA certificate
    debian使用过程中常见的问题
    将普通用户添加到sudo
    nano
    jenkins安装和使用
  • 原文地址：https://www.cnblogs.com/bchjazh/p/14472742.html
Copyright © 2011-2022 走看看