sql 马来西亚房地产展2018年12月

Posted 2021-05-13

-- First Query
-- To compare the number of view of properties in each states of Malaysia from Singapore visitor


SELECT
  EXTRACT(YEAR FROM _PARTITIONTIME) AS year,
  EXTRACT(MONTH FROM _PARTITIONTIME) AS month,
  region_code,
  my_geo.description,
  property_type_group_name,
  COUNT(DISTINCT full_visitor_id) as num_of_visitor,
  AVG(price) as average_of_price,
  MIN(price) as min_of_price,
  MAX(price) as max_of_price
FROM
  `propertyguru-datalake-v0.ods.my_seeker_events_denormed` my_seeker
  INNER JOIN `propertyguru-datalake-v0.propertydb_staging.MY_dump_geographic_area_production` my_geo
  ON my_seeker.region_code = my_geo.code
WHERE
  _PARTITIONTIME >= "2017-01-01 00:00:00"
--   AND _PARTITIONTIME < "2018-11-22 00:00:00"
  AND city = 'Singapore'
  AND vlisting_id IS NOT NULL
  AND vlisting_id != '0'
  AND my_seeker.type_code = 'SALE'
  -- Remove outlier
  AND price > 130000
  AND price < 3059000
--   AND code IN ('MY01','MY14','MY04','MY07')
GROUP BY 1,2,3,4,5
ORDER BY 1,2,3,5


-- Break down by Price point
SELECT
  EXTRACT(YEAR FROM _PARTITIONTIME) AS year,
--   EXTRACT(MONTH FROM _PARTITIONTIME) AS month,
  region_code,
  my_geo.description,
  property_type_group_name,
  COUNT(CASE WHEN price < 1000000 THEN 1 ELSE NULL END) AS less_than_1mil,
  COUNT(CASE WHEN price >= 1000000 AND price < 1300000 THEN 1 ELSE NULL END) AS a1milto1mil3,
  COUNT(CASE WHEN price >= 1300000 AND price < 1700000 THEN 1 ELSE NULL END) AS a1mil3to1mil7,
  COUNT(CASE WHEN price >= 1700000 AND price < 2000000 THEN 1 ELSE NULL END) AS a1mil7to2mil,
  COUNT(CASE WHEN price >= 2000000 THEN 1 ELSE NULL END) AS more_than_2mil
FROM
  `propertyguru-datalake-v0.ods.my_seeker_events_denormed` my_seeker
  INNER JOIN `propertyguru-datalake-v0.propertydb_staging.MY_dump_geographic_area_production` my_geo
  ON my_seeker.region_code = my_geo.code
WHERE
  _PARTITIONTIME >= "2017-01-01 00:00:00"
--   AND _PARTITIONTIME < "2018-11-22 00:00:00"
  AND city = 'Singapore'
  AND vlisting_id IS NOT NULL
  AND vlisting_id != '0'
  AND my_seeker.type_code = 'SALE'
  AND price > 130000
  AND price < 3059000
--   AND code IN ('MY01','MY14','MY04','MY07')
GROUP BY 1,2,3,4
ORDER BY 1,2,3

-- Break down by floor area
SELECT
  EXTRACT(YEAR FROM _PARTITIONTIME) AS year,
  EXTRACT(MONTH FROM _PARTITIONTIME) AS month,
  region_code,
  my_geo.description,
  property_type_group_name,
  COUNT(CASE WHEN floorarea < 1000 THEN 1 ELSE NULL END) AS less_than_1k,
  COUNT(CASE WHEN floorarea >= 1000 AND price < 1300 THEN 1 ELSE NULL END) AS a1kto1k3,
  COUNT(CASE WHEN floorarea >= 1300 AND price < 1600 THEN 1 ELSE NULL END) AS a1k3to1k6,
  COUNT(CASE WHEN floorarea >= 1600 AND price < 2000 THEN 1 ELSE NULL END) AS a16to2k,
  COUNT(CASE WHEN price >= 2000 THEN 1 ELSE NULL END) AS more_than_2k
FROM
  `propertyguru-datalake-v0.ods.my_seeker_events_denormed` my_seeker
  INNER JOIN `propertyguru-datalake-v0.propertydb_staging.MY_dump_geographic_area_production` my_geo
  ON my_seeker.region_code = my_geo.code
WHERE
  _PARTITIONTIME >= "2017-01-01 00:00:00"
--   AND _PARTITIONTIME < "2018-11-22 00:00:00"
  AND city = 'Singapore'
  AND vlisting_id IS NOT NULL
  AND vlisting_id != '0'
  AND my_seeker.type_code = 'SALE'
  AND floorarea > 527
  AND price < 7363
--   AND code IN ('MY01','MY14','MY04','MY07')
GROUP BY 1,2,3,4,5
ORDER BY 1,2,3,4



-- Find Percentile 

SELECT
   APPROX_QUANTILES(floorarea , 50) AS approx_quantiles

FROM
  `propertyguru-datalake-v0.propertydb_staging.my_v_listing`
   WHERE type_code = 'SALE'
   AND floorarea IS NOT NULL
   AND floorarea != 0



---------------------
-- Testing Section --
---------------------

CREATE TEMPORARY FUNCTION
 last_n(x STRING,
   n FLOAT64,
   delimiter STRING)
 RETURNS STRING
 LANGUAGE js AS """
         if (x == null ) {return null} else if (x.split(delimiter).slice(-n,-1) == "") {return x.split(delimiter).pop()} else {return [x.split(delimiter).slice(-n,-1), x.split(delimiter).pop()]};
       """;

SELECT
  hits_pagepath,
  last_n(hits_pagepath,1, '-') as listing_id,
  vlisting_id,
  hits_pagepath_level1, 
  city
FROM
  `propertyguru-datalake-v0.ods.my_seeker_events_denormed`
WHERE
  _PARTITIONTIME >= "2018-11-19 00:00:00"
  AND _PARTITIONTIME < "2018-11-22 00:00:00"
  AND city = 'Singapore'
LIMIT
  1000


SELECT
id,
region_code
FROM
  `propertyguru-datalake-v0.propertydb_staging.my_v_listing`
LIMIT
  1000



Can get in MY_dump..... table
  MY01 - JH  <---
  MY02 - KD
  MY03 - KT
  MY04 - ML  <---
  MY05 - NS
  MY06 - PH
  MY07 - PG  <---
  MY08 - PR 
  MY09 - PS 
  MY10 - SL 
  MY11 - TR 
  MY12 - SB 
  MY13 - SW
  MY14 - KL  <---


CREATE TEMPORARY FUNCTION
 last_n(x STRING,
   n FLOAT64,
   delimiter STRING)
 RETURNS STRING
 LANGUAGE js AS """
         if (x == null ) {return null} else if (x.split(delimiter).slice(-n,-1) == "") {return x.split(delimiter).pop()} else {return [x.split(delimiter).slice(-n,-1), x.split(delimiter).pop()]};
       """;

WITH
  seeker_tb AS (
    SELECT
      last_n(hits_pagepath,1, '-') as listing_id,
      city
    FROM
      `propertyguru-datalake-v0.ods.my_seeker_events_denormed`
    WHERE
      _PARTITIONTIME >= "2018-11-20 00:00:00"
      AND _PARTITIONTIME < "2018-11-21 00:00:00"
      AND city = 'Singapore'
     group by 1,2
    ),
   listing_tb AS (
    SELECT
    CAST(id as STRING) as id,
    region_code
    FROM
      `propertyguru-datalake-v0.propertydb_staging.my_v_listing`
--     group by 1,2
   )

SELECT
id,
region_code
FROM listing_tb
  INNER JOIN seeker_tb
  ON seeker_tb.listing_id = listing_tb.id
LIMIT 1000






-- CREATE TEMPORARY FUNCTION
--  last_n(x STRING,
--    n FLOAT64,
--    delimiter STRING)
--  RETURNS STRING
--  LANGUAGE js AS """
--          if (x == null ) {return null} else if (x.split(delimiter).slice(-n,-1) == "") {return x.split(delimiter).pop()} else {return [x.split(delimiter).slice(-n,-1), x.split(delimiter).pop()]};
--        """;

SELECT
  hits_pagepath,
   SPLIT(hits_pagepath, '-')[SAFE_OFFSET(ARRAY_LENGTH(SPLIT(hits_pagepath, '-')) - 1)] as listing_id,
--   last_n(hits_pagepath,1, '-') as listing_id,
  vlisting_id,
  hits_pagepath_level1, 
  city,
  status_code
FROM
  `propertyguru-datalake-v0.ods.my_seeker_events_denormed`
WHERE
  _PARTITIONTIME >= "2018-11-19 00:00:00"
  AND _PARTITIONTIME < "2018-11-22 00:00:00"
  AND city = 'Singapore'