使用avro序列化和反序列化

Posted 2022-12-17 xinyuan_java

mvn 引入需要的jar包

        
        <dependency>
		  <groupId>org.apache.avro</groupId>
		  <artifactId>avro</artifactId>
		  <version>1.11.1</version>
		</dependency>
		<dependency>
		    <groupId>org.xerial.snappy</groupId>
		    <artifactId>snappy-java</artifactId>
		    <version>1.1.8.4</version>
		</dependency>

 编辑对应的定义

person.avsc

    "namespace":"org.cloud.avro",
    "type":"record",
    "name":"Person",
    "fields":[
        "name":"id","type":"string",
        "name":"name","type":"string",
        "name":"age","type":["int","null"]
    ]

生成对应的model

1. 借助 avro-tools 生成

2. 借助 avro-maven-plugin 执行 mvn clean isntall 生成

<plugin>
  <groupId>org.apache.avro</groupId>
  <artifactId>avro-maven-plugin</artifactId>
  <version>1.11.1</version>
  <executions>
    <execution>
      <phase>generate-sources</phase>
      <goals>
        <goal>schema</goal>
      </goals>
      <configuration>
        <sourceDirectory>$project.basedir/src/main/resources/avsc</sourceDirectory>
        <outputDirectory>$project.basedir/src/main/java/</outputDirectory>
      </configuration>
    </execution>
  </executions>
</plugin>
<plugin>
  <groupId>org.apache.maven.plugins</groupId>
  <artifactId>maven-compiler-plugin</artifactId>
  <configuration>
    <source>1.8</source>
    <target>1.8</target>
  </configuration>
</plugin>

main sample

package org.cloud.avro;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.avro.file.DataFileReader;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.specific.SpecificDatumWriter;

import cn.hutool.json.JSONUtil;

public class AvroMain 

    public static void write() throws IOException 
        List<Person> persons = new ArrayList<>();
        DatumWriter<Person> dw = new SpecificDatumWriter<>(Person.class);
        DataFileWriter<Person> dfw = new DataFileWriter<>(dw);
        dfw.create(Person.SCHEMA$, new File("d://file//avro/person.avro"));
        for (int i = 0; i < 100000; i++) 
            Person person = new Person("00" + i, "zhangsan", 23);
            persons.add(person);
            dfw.append(person);
        
        dfw.close();

        String str = JSONUtil.toJsonStr(persons);
        FileWriter fileWriter = new FileWriter("d://file//avro/person.json");
        fileWriter.write(str);
        fileWriter.flush();
        fileWriter.close();

    

    public static void read() throws IOException 
        DatumReader<Person> dr = new SpecificDatumReader<Person>(Person.class);
        DataFileReader<Person> dfr = new DataFileReader<Person>(new File("d://file//avro/person.avro"), dr);
        Person person = null;
        while (dfr.hasNext()) 
            person = dfr.next();
            System.out.println(person);
        

    

    public static void main(String[] args) 
        try 
            write();
         catch (Exception e) 
            e.printStackTrace();
        
        System.out.println("done ");
    

学习： avro数据序列化/反序列化 https://blog.csdn.net/lianghecai52171314/article/details/104241123