千家信息网

flink 从mysql 读取数据 放入kafka中 用于搜索全量

发表于:2024-10-21 作者:千家信息网编辑
千家信息网最后更新 2024年10月21日,接着上一篇,将mysql的数据导入kafka中public static void main(String[] arg) throws Exception { TypeInformati
千家信息网最后更新 2024年10月21日flink 从mysql 读取数据 放入kafka中 用于搜索全量

接着上一篇,将mysql的数据导入kafka中

public static void main(String[] arg) throws Exception {        TypeInformation[] fieldTypes = new TypeInformation[] { BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO };        String[] fieldNames = new String[] { "name", "address" };        RowTypeInfo rowTypeInfo = new RowTypeInfo(fieldTypes, fieldNames);        JDBCInputFormat jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat().setDrivername("com.mysql.jdbc.Driver")                .setDBUrl("jdbc:mysql://ip:3306/tablespace?characterEncoding=utf8")                .setUsername("user").setPassword("root")                .setQuery("select LOGIC_CODE, SHARE_LOG_CODE from table").setRowTypeInfo(rowTypeInfo).finish();        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();        DataSource s = env.createInput(jdbcInputFormat);        BatchTableEnvironment tableEnv = new BatchTableEnvironment(env, TableConfig.DEFAULT());        tableEnv.registerDataSet("t2", s);        Table tapiResult = tableEnv.scan("t2");        System.out.println("schema is:");        tapiResult.printSchema();        Table query = tableEnv.sqlQuery("select name, address from t2");        DataSet ds=  tableEnv.toDataSet(query, Result.class);        DataSet temp=ds.map(new MapFunction() {            @Override            public String map(Result result) throws Exception {                String name = result.name;                String value = result.address;                return name+":->:"+value;            }        });        logger.info("read db end");         KafkaOutputFormat kafkaOutput = KafkaOutputFormat.buildKafkaOutputFormat()                .setBootstrapServers("ip:9092").setTopic("search_test_whk").setAcks("all").setBatchSize("1000")                .setBufferMemory("100000").setLingerMs("1").setRetries("2").finish();        temp.output(kafkaOutput);        logger.info("write kafka end");        env.execute("Flink add data source");    }
0