Spark 写Hive指定动态分区

1、设置

   SparkSession.Builder builder = SparkSession
                .builder()
                .config(conf)
                .master("yarn")
                .config("hive.exec.dynamici.partition", true)
                .config("hive.exec.dynamic.partition.mode", "nonstrict")
                .appName(taskName)
                .enableHiveSupport();

　　

2、sql
　　

  /**
     * 同步数据到clickhouse
     * 按照日期分区
     *
     * @param dataFrame
     */
    private void sinkToHive(Dataset<Row> dataFrame) {
        dataFrame.createOrReplaceTempView("soh_tmp");
        String selectSql = "select " + getSelectHiveColumn() + " from soh_tmp";
        String sql = "INSERT INTO TABLE d_vehicle_data_charging_u_d partition(part_key) " + selectSql;
        sparkSession.sql(sql);
    }

    /**
     * @return
     */
    private String getSelectHiveColumn() {
        return
                "vin,\n" +
                        "batch,\n" +
                        "tbox_type,\n" +
                        "collect_time,\n" +
                        "`current_timestamp` AS receive_time,\n" +
                        "battery_voltage_data,\n" +
                        "battery_temp_data,\n" +
                        "charging_status,\n" +
                        "total_current,\n" +
                        "battery_soc,\n" +
                        "max_temp,\n" +
                        "min_temp,\n" +
                        "partition_key,\n" +
                        "total_mileage,\n" +
                        "max_cell_voltage,\n" +
                        "min_cell_voltage,\n" +
                        "level_alarm,\n" +
                        "max_voltage_bat_sys_no,\n" +
                        "max_voltage_bat_cell_no," +
                        "mix_voltage_bat_sys_no,\n" +
                        "min_voltage_bat_cell_no,\n" +
                        "vehicle_status,\n" +
                        "insulation_resistance,\n" +
                        "insulation_Alarm, " +
                        "`current_timestamp`() as create_time,\n" +
                        " partition_key as part_key\n"
                ;
    }