sh cca175-problem-03-evolve-avro-schema.sh

Posted 2021-05-07

use retail;
select * from orders_avro as X where X.order_date in (
  select inner.order_date from (
    select Y.order_date, count(1) as total_orders 
      from orders_avro as Y 
      group by Y.order_date 
      order by total_orders desc, Y.order_date desc 
      limit 1
  ) i
);


// Evolve Avro Schema  
// 1. Get schema file
hadoop fs -get /user/hive/schemas/order/orders.avsc
// 2. Open schema file
gedit orders.avsc
// 3. Edit schema file
{
  "type" : "record",
  "name" : "orders",
  "doc" : "Sqoop import of orders",
  "fields" : [ {
    "name" : "order_id",
    "type" : [ "null", "int" ],
    "default" : null,
    "columnName" : "order_id",
    "sqlType" : "4"
  }, {
    "name" : "order_date",
    "type" : [ "null", "long" ],
    "default" : null,
    "columnName" : "order_date",
    "sqlType" : "93"
  }, {
    "name" : "order_customer_id",
    "type" : [ "null", "int" ],
    "default" : null,
    "columnName" : "order_customer_id",
    "sqlType" : "4"
  },{
    "name" : "order_style",
    "type" : [ "null", "string" ],
    "default" : null,
    "columnName" : "order_style",
    "sqlType" : "12"
  }, {
    "name" : "order_zone",
    "type" : [ "null", "int" ],
    "default" : null,
    "columnName" : "order_zone",
    "sqlType" : "4"
  }, {
    "name" : "order_status",
    "type" : [ "null", "string" ],
    "default" : null,
    "columnName" : "order_status",
    "sqlType" : "12"
  } ],
  "tableName" : "orders"
}

// 3. copy modified schema file to HDFS again
hadoop fs -copyFromLocal -f orders.avsc /user/hive/schemas/order/orders.avsc