sh sqoop-incremental-load.sh

Posted 2021-05-07

# Baseline import
sqoop import \
  --connect jdbc:mysql://ms.itversity.com:3306/retail_db \
  --username retail_user \
  --password itversity \
  --target-dir /user/dgadiraju/sqoop_import/retail_db/orders \
  --num-mappers 2 \
  --query "select * from orders where \$CONDITIONS and order_date like '2013-%'" \
  --split-by order_id

# Query can be used to load data based on condition
sqoop import \
  --connect jdbc:mysql://ms.itversity.com:3306/retail_db \
  --username retail_user \
  --password itversity \
  --target-dir /user/dgadiraju/sqoop_import/retail_db/orders \
  --num-mappers 2 \
  --query "select * from orders where \$CONDITIONS and order_date like '2014-01%'" \
  --split-by order_id \
  --append

# where in conjunction with table can be used to get data based up on a condition
sqoop import \
  --connect jdbc:mysql://ms.itversity.com:3306/retail_db \
  --username retail_user \
  --password itversity \
  --target-dir /user/dgadiraju/sqoop_import/retail_db/orders \
  --num-mappers 2 \
  --table orders \
  --where "order_date like '2014-02%'" \
  --append

# Incremental load using arguments specific to incremental load
sqoop import \
  --connect jdbc:mysql://ms.itversity.com:3306/retail_db \
  --username retail_user \
  --password itversity \
  --target-dir /user/dgadiraju/sqoop_import/retail_db/orders \
  --num-mappers 2 \
  --table orders \
  --check-column order_date \
  --incremental append \
  --last-value '2014-02-28'