基于pg_qualstats和hypopg的自动索引调优

Posted xueqiuqiu

tags:

篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了基于pg_qualstats和hypopg的自动索引调优相关的知识,希望对你有一定的参考价值。

pg-qualstats的安装和配置

1.安装pg-qualstats

sudo apt install postgresql-10-pg-qualstats

 

2.将pg_qualstats和pg_stat_statements添加到shared_preload_libraries,使得postgresql .conf文件中具有以下设置:

#shared_preload_libraries = pg_stat_statements,pg_qualstats # (change requires restart)

postgresql.conf文件在/etc/postgresql/10/main/目录下

执行grep ‘shared_preload‘ postgresql.conf可见

 技术图片

 

 

 3.重新启动PG

service postgresql restart

4.进入PG

sudo su - postgres

psql

5.查询shared_preload_libraries

show shared_preload_libraries ;

技术图片

 

 

 

 

Hypopg的安装和配置

1.安装 hypopg

apt install postgresql-server-dev-10 

apt install postgresql-10-hypopg

 

 

自动索引调优

1.进入PG

sudo su - postgres

2.设置采样率 pg_qualstats .sample_rate1,保证调参涉及到所有的query

psql -d postgres -c "ALTER SYSTEM SET pg_qualstats.sample_rate TO 1"

验证

psql -c "select pg_reload_conf()"

3.进入PG

psql

4.加载extension

CREATE EXTENSION hypopg;

CREATE EXTENSION pg_stat_statements ;

CREATE EXTENSION pg_qualstats;

5.查看配置

dx

show shared_preload_libraries ;

 6.建立测试数据库

create database testdb owner postgres;

7.复现样例测试

建表

CREATE TABLE test (id int, dept int, id2 int, id3 int, id4 int, id5 int,id6 int,id7 int,details text, zipcode int);

插入数据

INSERT INTO test SELECT (random() * 1000000)::int, (random() * 1000000)::int, (random() * 1000000)::int,(random() * 1000000)::int,(random() * 1000000)::int,(random() * 1000000)::int,(random() * 1000000)::int,(random() * 1000000)::int,

md5(g::text), floor(random()* (20000-9999 + 1) + 9999)

FROM generate_series(1,1*1e6) g;

执行workload

select * from test where id2 = 1 and id4 = 3;

select * from test where id3 = 3;

select * from test where id3 = 3 and id4 = 2;

select * from test where id4 = 2 and id2 = 3;

建立函数 find_usable_indexes

CREATE OR REPLACE FUNCTION find_usable_indexes()

RETURNS VOID AS

$$

DECLARE

    l_queries     record;

    l_querytext     text;

    l_idx_def       text;

    l_bef_exp       text;

    l_after_exp     text;

    hypo_idx      record;

    l_attr        record;

    /* l_err       int; */

BEGIN

    CREATE TABLE IF NOT EXISTS public.idx_recommendations (queryid bigint,

    query text, current_plan jsonb, recmnded_index text, hypo_plan jsonb);

    FOR l_queries IN

    SELECT t.relid, t.relname, t.queryid, t.attnames, t.attnums,

    pg_qualstats_example_query(t.queryid) as query

      FROM

        (

         SELECT qs.relid::regclass AS relname, qs.relid AS relid, qs.queryid,

         string_agg(DISTINCT attnames.attnames,,) AS attnames, qs.attnums

         FROM pg_qualstats_all qs

         JOIN pg_qualstats q ON q.queryid = qs.queryid

         JOIN pg_stat_statements ps ON q.queryid = ps.queryid

         JOIN pg_amop amop ON amop.amopopr = qs.opno

         JOIN pg_am ON amop.amopmethod = pg_am.oid,

         LATERAL

              (

               SELECT pg_attribute.attname AS attnames

               FROM pg_attribute

               JOIN unnest(qs.attnums) a(a) ON a.a = pg_attribute.attnum

               AND pg_attribute.attrelid = qs.relid

               ORDER BY pg_attribute.attnum) attnames,     

         LATERAL unnest(qs.attnums) attnum(attnum)

               WHERE NOT

               (

                EXISTS

                      (

                       SELECT 1

                       FROM pg_index i

                       WHERE i.indrelid = qs.relid AND

                       (arraycontains((i.indkey::integer[])[0:array_length(qs.attnums, 1) - 1],

                        qs.attnums::integer[]) OR arraycontains(qs.attnums::integer[],

                        (i.indkey::integer[])[0:array_length(i.indkey, 1) + 1]) AND i.indisunique)))

                       GROUP BY qs.relid, qs.queryid, qs.qualnodeid, qs.attnums) t

                       GROUP BY t.relid, t.relname, t.queryid, t.attnames, t.attnums                   

    LOOP

        /* RAISE NOTICE ‘% : is queryid‘,l_queries.queryid; */

        execute explain (FORMAT JSON) ||l_queries.query INTO l_bef_exp;

        execute select hypopg_reset();

        execute SELECT indexrelid,indexname FROM hypopg_create_index(‘‘CREATE INDEX on ||l_queries.relname||(||l_queries.attnames||)‘‘) INTO hypo_idx;      

        execute explain (FORMAT JSON) ||l_queries.query INTO l_after_exp;

        execute select hypopg_get_indexdef(||hypo_idx.indexrelid||) INTO l_idx_def;

        INSERT INTO public.idx_recommendations (queryid,query,current_plan,recmnded_index,hypo_plan)

        VALUES (l_queries.queryid,l_querytext,l_bef_exp::jsonb,l_idx_def,l_after_exp::jsonb);        

    END LOOP;    

        execute select hypopg_reset();

END;

$$ LANGUAGE plpgsql;

执行函数find_usable_indexes

select find_usable_indexes();

查找索引

select queryid, current_plan->0->Plan->>Total Cost as "cost_without_index",

hypo_plan->0->Plan->>Total Cost as "cost_with_index",

round((((current_plan->0->Plan->>Total Cost)::numeric-(hypo_plan->0->Plan->>Total Cost)::numeric)*100/(current_plan->0->Plan->>Total Cost)::numeric),2) as percent_improvd

FROM idx_recommendations order by 4 desc;

 

 

select b.query, a.recmnded_index,round((((a.current_plan->0->Plan->>Total Cost)::numeric-(hypo_plan->0->Plan->>Total Cost)::numeric)*100/(a.current_plan->0->Plan->>Total Cost)::numeric),2) as percent_improvd FROM idx_recommendations a JOIN pg_stat_statements b ON a.queryid = b.queryid WHERE round((((current_plan->0->Plan->>Total Cost)::numeric-(hypo_plan->0->Plan->>Total Cost)::numeric)*100/(current_plan->0->Plan->>Total Cost)::numeric),2) > 0 order by 3 desc ;

 

 

以上是关于基于pg_qualstats和hypopg的自动索引调优的主要内容,如果未能解决你的问题,请参考以下文章

pgspider 集成hypopg 虚拟索引扩展

虚拟索引插件之HypoPG

虚拟索引插件之HypoPG

wordpress 是不是通过自动索引来优化其数据库表?

网络公开课总结自动收索效果

一种自动分类数据方法_初探索