Kettle系列:使用Kudu API插入数据到Kudu中

Posted 2020-11-01 harrychinese 刘忠武

本文详细介绍了在Kettle中使用 Kudu API将数据写入Kudu中, 从本文可以学习到:
1. 如何编写一个简单的 Kettle 的 Used defined Java class.
2. 如何读取Kettle 每个记录的字段. 需要注意的是 getInteger() 返回的是Long 对象; 而获取 Timestamp 字段的方法是getDate().
3. 如何调用Kudu API.

本Kettle示例非常简单, Data Grid 组件定义一些sample data(包含多种数据类型), Java class将这些sample data写入kudu. 

Kudu表schema:

CREATE TABLE kudu_testdb.perf_test_t1
( 
    id string ENCODING PLAIN_ENCODING COMPRESSION SNAPPY,
    int_value int,
    bigint_value bigint, 
    timestamp_value timestamp, 
    bool_value int,
    PRIMARY KEY (histdate,id)  
) 
PARTITION BY HASH (histdate,id) PARTITIONS 2 
STORED AS KUDU
TBLPROPERTIES (
  \'kudu.table_name\' = \'testdb.perf_test_t1\',
  \'kudu.master_addresses\' = \'10.205.6.1:7051,10.205.6.2:7051,10.205.7.3:7051\'
);

 

重点看Java class 代码:

import java.sql.Timestamp;
import java.util.UUID;
import static java.lang.Math.toIntExact;

import org.apache.kudu.client.Insert;
import org.apache.kudu.client.KuduClient;
import org.apache.kudu.client.KuduException;
import org.apache.kudu.client.KuduSession;
import org.apache.kudu.client.KuduTable;
import org.apache.kudu.client.PartialRow;
import org.apache.kudu.client.SessionConfiguration;

private final static String KUDU_TABLE="testdb.perf_test_t1";
private final static String KUDU_SERVERS="10.205.6.1:7051,10.205.6.2:7051,10.205.7.3:7051";
private final static int OPERATION_BATCH = 50;  

KuduClient client=null;
KuduSession session=null;
KuduTable table=null;
Integer recordCount=null;
SessionConfiguration.FlushMode mode;

private Object[] previousRow;
private Object[] currentRow;


public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
  if (first) {
    first = false; 
  }

  currentRow = getRow();
  if (currentRow == null) {
    setOutputDone();
    return false;
  }
 
    try {
        session.setFlushMode(mode);
        session.setMutationBufferSpace(OPERATION_BATCH);

        int uncommit = 0;
        while(currentRow != null) {
            Insert insert = table.newInsert();
            PartialRow kuduRow = insert.getRow();
                       
            int intTmp;
            Long longTmp;
            String stringTmp;
            java.util.Date dateTmp;
            Boolean booleanTmp;
            
                
            // kettle string -> kudu string 
            //kuduRow.addString("id",UUID.randomUUID().toString());
            stringTmp =  get(Fields.In, "id").getString(currentRow);    
            if (stringTmp!=null)
            {
               kuduRow.addString("id",stringTmp);
            }
            
            // kettle int -> kudu int
            //import static java.lang.Math.toIntExact;
            longTmp=get(Fields.In, "int_value").getInteger(currentRow);
            if (longTmp!=null)
            {
               intTmp =toIntExact(get(Fields.In, "int_value").getInteger(currentRow)); 
               kuduRow.addInt("int_value",  intTmp);
            } 
            

            // kettle bigint -> kudu bigint 
            longTmp=get(Fields.In, "bigint_value").getInteger(currentRow);
            if (longTmp!=null)
            { 
                 kuduRow.addLong("bigint_value", longTmp);
             }
 
            // kettle date/timestamp  -> kudu timestamp 
            dateTmp= get(Fields.In, "timestamp_value").getDate(currentRow);
            if (dateTmp!=null)
            { 
               longTmp=dateTmp.getTime()+8*3600*1000; //转到东8区时间
               kuduRow.addLong("timestamp_value", longTmp*1000);             
            } 
    
 
            // kettle boolean  -> kudu int
            booleanTmp= get(Fields.In, "boolean_value").getBoolean(currentRow);
            if (booleanTmp!=null)
            { 
               intTmp=0;
               if (booleanTmp)
                {intTmp=1;}         
               kuduRow.addInt("boolean_value", intTmp);             
            } 
           
            // 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
             uncommit = uncommit + 1;
                if (uncommit > OPERATION_BATCH / 2) {
                    session.flush();
                     uncommit = 0;
                }
            session.apply(insert);
            previousRow=currentRow;
            currentRow=getRow();
        }

        // 对于手工提交, 保证完成最后的提交
        if (uncommit > 0) {
            session.flush();
        }

       } catch (Exception e) {
        e.printStackTrace();
        throw e; 
     }

  // Send the row on to the next step.
  //putRow(data.outputRowMeta, currentRow);

  return false;
}

public boolean init(StepMetaInterface stepMetaInterface, StepDataInterface stepDataInterface) {
  try {
         client = new KuduClient.KuduClientBuilder(KUDU_SERVERS).build();
         session = client.newSession();      
         table =client.openTable(KUDU_TABLE);
         mode = SessionConfiguration.FlushMode.MANUAL_FLUSH;
       } catch (Exception e) {
        e.printStackTrace();
        throw e; 
     }

  return parent.initImpl(stepMetaInterface, stepDataInterface);
}

public void dispose(StepMetaInterface smi, StepDataInterface sdi) {
    try {
            if (!session.isClosed()) {
                session.close();
            }
       } catch (Exception e) {
        e.printStackTrace();
        throw e; 
     }
  parent.disposeImpl(smi, sdi);
}