zoukankan      html  css  js  c++  java
  • flink-conf.yaml

    Flink 配置文件 对于管理员来说,差不多经常调整的就只有 conf 下的flink-conf.yaml : 经过初步的调整,大约有以下模块的参数(未优化)

    ################################################################################

    Licensed to the Apache Software Foundation (ASF) under one

    or more contributor license agreements. See the NOTICE file

    distributed with this work for additional information

    regarding copyright ownership. The ASF licenses this file

    to you under the Apache License, Version 2.0 (the

    "License"); you may not use this file except in compliance

    with the License. You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software

    distributed under the License is distributed on an "AS IS" BASIS,

    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

    See the License for the specific language governing permissions and

    limitations under the License.

    ################################################################################

    ==============================================================================

    Jobmanager && TaskManager 的基本参数

    ==============================================================================

    jobmanager.rpc.address: flink-master

    jobmanager.rpc.port: 6123
    jobmanager.heap.size: 32768m
    jobstore.cache-size: 52428800
    taskmanager.heap.size: 92160m
    taskmanager.numberOfTaskSlots: 30
    parallelism.default: 3
    io.tmp.dirs: /data1/data/flink/local
    taskmanager.network.numberOfBuffers: 4096
    taskmanager.memory.fraction: 0.7
    taskmanager.memory.preallocate: false
    taskmanager.debug.memory.startLogThread: 8
    taskmanager.debug.memory.logIntervalMs: 3600000
    blob.fetch.retries: 50
    blob.fetch.num-concurrent: 200
    blob.fetch.backlog: 500
    task.cancellation-interval: 30000
    taskmanager.exit-on-fatal-akka-error: false
    taskkmanager.log.path:

    ==============================================================================

    High Availability && and checkpointing

    =============================================================================

    high-availability: ZOOKEEPER
    high-availability.cluster-id: /default_flink_dc
    high-availability.zookeeper.quorum: VECS01455:2181,VECS01456:2181,VECS01457:2181,VECS01458:2181,VECS01459:2181
    high-availability.storageDir: hdfs://flashHadoop/flink/recovery
    high-availability.zookeeper.path.root: /flink
    high-availability.zookeeper.path.checkpoint-counter: /checkpoint-counter
    high-availability.zookeeper.path.checkpoints: /checkpoints
    high-availability.zookeeper.path.jobgraphs: /jobgraphs
    high-availability.zookeeper.path.latch: /leaderlatch
    high-availability.zookeeper.path.leader: /leader
    high-availability.zookeeper.path.mesos-workers: /mesos-workers
    high-availability.zookeeper.path.running-registry: /running_job_registry/
    high-availability.zookeeper.client.connection-timeout: 18000
    high-availability.zookeeper.client.max-retry-attempts: 10
    high-availability.zookeeper.client.retry-wait: 5000
    high-availability.zookeeper.client.session-timeout: 60000

    ==============================================================================

    HDFS and Fault tolerance and checkpointing

    ==============================================================================

    fs.default-scheme: hdfs://flashHadoop/flink/data
    fs.output.always-create-directory: false
    fs.overwrite-files: false

    state.backend: filesystem
    state.backend.async: true
    state.backend.fs.memory-threshold: 4096
    state.backend.incremental: false
    state.backend.local-recovery: false
    state.checkpoints.dir: hdfs://flashHadoop/flink/checkpoints_data
    state.checkpoints.num-retained: 1
    state.savepoints.dir: hdfs://flashHadoop/flink/savepoints_data

    ==============================================================================

    Web Frontend

    ==============================================================================

    web.address: 0.0.0.0
    web.access-control-allow-origin: "*"
    web.backpressure.cleanup-interval: 600000
    web.backpressure.delay-between-samples: 50
    web.backpressure.num-samples: 100
    web.backpressure.refresh-interval: 60000
    web.checkpoints.history: 20
    web.history: 20
    web.log.path: /data1/data/flink/web/log
    web.refresh-interval: 3000
    web.submit.enable: true
    web.timeout: 10000
    web.tmpdir: /data1/data/flink/web/tmp

    ==============================================================================

    Flink Cluster Security Configuration

    ==============================================================================

    test is not set Security

    ==============================================================================

    HistoryServer Config

    ==============================================================================

    historyserver.web.address: 0.0.0.0
    historyserver.web.port: 8082
    historyserver.archive.fs.dir: hdfs://flashHadoop/flink/completed_jobs/
    historyserver.archive.fs.refresh-interval: 2000
    historyserver.web.tmpdir: /data1/data/flink/tmp

    ==============================================================================

    ==============================================================================

    Environment 角色环境 申明

    ==============================================================================

    env.java.opts: -XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+G1SummarizeConcMark -XX:MaxGCPauseMillis=100 -XX:-ResizePLAB -XX:+ParallelRefProcEnabled -XX:ParallelGCThreads=24 -XX:ConcGCThreads=16 -XX:G1HeapWastePercent=3 -XX:InitiatingHeapOccupancyPercent=35 -XX:G1MixedGCLiveThresholdPercent=85

    env.java.opts.historyserver: -Xmx16g -Xms16g -Xmn4g -XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+G1SummarizeConcMark -XX:MaxGCPauseMillis=100 -XX:-ResizePLAB -XX:+ParallelRefProcEnabled -XX:+AlwaysPreTouch -XX:ParallelGCThreads=24 -XX:ConcGCThreads=16 -XX:G1HeapWastePercent=3 -XX:InitiatingHeapOccupancyPercent=35 -XX:G1MixedGCLiveThresholdPercent=85 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:/data1/data/flink/gc/gc-historyserver-hostname.log

    env.java.opts.jobmanager: -XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+G1SummarizeConcMark -XX:MaxGCPauseMillis=100 -XX:-ResizePLAB -XX:+ParallelRefProcEnabled -XX:ParallelGCThreads=24 -XX:ConcGCThreads=16 -XX:G1HeapWastePercent=3 -XX:InitiatingHeapOccupancyPercent=35 -XX:G1MixedGCLiveThresholdPercent=85 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:/data1/data/flink/gc/gc-jobmanager-hostname.log

    env.java.opts.taskmanager: -XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+G1SummarizeConcMark -XX:MaxGCPauseMillis=100 -XX:-ResizePLAB -XX:+ParallelRefProcEnabled -XX:ParallelGCThreads=24 -XX:ConcGCThreads=16 -XX:G1HeapWastePercent=3 -XX:InitiatingHeapOccupancyPercent=35 -XX:G1MixedGCLiveThresholdPercent=85 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:/data1/data/flink/gc/gc-taskmanager-hostname.log

    env.log.dir: /data1/data/flink/log

    env.log.max: 1

  • 相关阅读:
    C# 日志本地化工具
    javascript面向对象的写法01
    VM12-Pro 安装CentOS7 并配置静态IP出坑记
    基于CentOS7.x安装Nginx-1.18.0
    程序员思维导图、web初学者必备、web前端知识集锦-不断更新中...
    js知识
    swiper的使用
    web 移动端键盘处理-vue移动端那些事
    vue学习计划-vuex生态
    vue 组件复用
  • 原文地址:https://www.cnblogs.com/hit-zb/p/10436840.html
Copyright © 2011-2022 走看看