zoukankan      html  css  js  c++  java
  • Flink状态之ReduceState

    1、主类

    package com.example.demo.flink;
    
    import com.example.demo.flink.impl.CountAverageWithMapState;
    import com.example.demo.flink.impl.CountAverageWithReduceState;
    import org.apache.flink.api.java.tuple.Tuple2;
    import org.apache.flink.configuration.Configuration;
    import org.apache.flink.streaming.api.datastream.DataStreamSource;
    import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
    
    
    /**
     * @program: demo
     * @description: valuestate
     * @author: yang
     * @create: 2020-12-28 15:46
     */
    public class TestKeyedReduceStateMain {
        public static void main(String[] args) throws  Exception{
            //获取执行环境
            StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(new Configuration());
            //StreamExecutionEnvironment.getExecutionEnvironment();
            //设置并行度
            env.setParallelism(16);
            //获取数据源
            DataStreamSource<Tuple2<Long, Long>> dataStreamSource =
                    env.fromElements(
                            Tuple2.of(1L, 3L),
                            Tuple2.of(1L, 7L),
                            Tuple2.of(2L, 4L),
                            Tuple2.of(1L, 5L),
                            Tuple2.of(2L, 2L),
                            Tuple2.of(2L, 6L));
    
    
            // 输出:
            //(1,5.0)
            //(2,4.0)
            dataStreamSource
                    .keyBy(0)
                    .flatMap(new CountAverageWithReduceState())
                    .print();
    
    
            env.execute("TestStatefulApi");
        }
    
    }

    2、处理实现类

    package com.example.demo.flink.impl;
    
    /**
     * @program: demo
     * @description: valuestate
     * @author: yang
     * @create: 2020-12-28 16:26
     */
    
    import org.apache.flink.api.common.functions.ReduceFunction;
    import org.apache.flink.api.common.functions.RichFlatMapFunction;
    import org.apache.flink.api.common.state.MapState;
    import org.apache.flink.api.common.state.MapStateDescriptor;
    import org.apache.flink.api.common.state.ReducingState;
    import org.apache.flink.api.common.state.ReducingStateDescriptor;
    import org.apache.flink.api.common.typeinfo.Types;
    import org.apache.flink.api.java.tuple.Tuple2;
    import org.apache.flink.configuration.Configuration;
    import org.apache.flink.shaded.guava18.com.google.common.collect.Lists;
    import org.apache.flink.util.Collector;
    
    import java.util.List;
    import java.util.UUID;
    
    /**
     *  ValueState<T> :这个状态为每一个 key 保存一个值
     *      value() 获取状态值
     *      update() 更新状态值
     *      clear() 清除状态
     *
     *      IN,输入的数据类型
     *      OUT:数据出的数据类型
     */
    public class CountAverageWithReduceState
            extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
    
        private ReducingState<Long> reducingState;
    
        /***状态初始化*/
        @Override
        public void open(Configuration parameters) throws Exception {
    
            ReducingStateDescriptor descriptor = new ReducingStateDescriptor("ReducingDescriptor", new ReduceFunction<Long>() {
                @Override
                public Long reduce(Long v1, Long v2) throws Exception {
                    return v1 + v2;
                }
            },Long.class);
            reducingState = getRuntimeContext().getReducingState(descriptor);
        }
    
        @Override
        public void flatMap(Tuple2<Long, Long> element, Collector<Tuple2<Long, Long>> collector) throws Exception {
    
            //将状态放入
            reducingState.add(element.f1);
            collector.collect(Tuple2.of(element.f0,reducingState.get()));
    
        }
    
    
    }
  • 相关阅读:
    使用XmlDataDocument将数据存储到XML文档
    Dubbo+Nacos 运行异常io.netty.handler.codec.EncoderException: java.lang.NullPointerException
    一、字符串常量Java内部加载
    更高效率,Java快速获取图片尺寸宽高分辨率(支持webp)
    水印宽高比例自适应图片(Java-Thumbnails)
    UEditor 自定义图片视频尺寸校验
    应用宝官方错误日志
    Eclipse使用hibernate插件
    解决Eclipse建立Maven项目后无法建立src/main/java资源文件夹的办法
    [Err] 1022
  • 原文地址:https://www.cnblogs.com/ywjfx/p/14228581.html
Copyright © 2011-2022 走看看