python pyspark-sql-hive-context.py

Posted 2021-05-09

# from pyspark.sql import HiveContext
# sqlContext = new HiveContext(sc);
sqlContext.setConf("spark.sql.shuffle.partitions", "2")
sqlContext.sql("use sparkdemo")

sql = """select o.order_date, sum(oi.order_item_subtotal) daily_revenue
        from orders o join order_items oi 
        on o.order_id = oi.order_item_order_id
        where o.order_status = 'COMPLETE' 
        group by o.order_date 
        order by o.order_date"""

sqlContext.sql(sql).show()

sqlContext.sql("create table daily_revenue(order_date string, daily_revenue float)")

i = "insert into daily_revenue " + sql
sqlContext.sql(i)

sqlContext.sql("select * from daily_revenue").show()