elasticsearch 异常处理 ——生命周期管理

Posted 2021-09-04 一曲广陵散

一、参考

elasticsearch 学习系列目录——更新ing

Index management error: “Index is not the write index for alias”

二、问题描述

GET _cat/tasks?v

action                         task_id                          parent_task_id                   type      start_time    timestamp running_time ip        node
cluster:monitor/tasks/lists    LrCxElELQGqRNI4tok60ug:234640577 -                                transport 1625046522112 09:48:42  175.8micros  127.0.0.1 ft-elasticsearch-8554868c8d-qzrck
cluster:monitor/tasks/lists[n] LrCxElELQGqRNI4tok60ug:234640578 LrCxElELQGqRNI4tok60ug:234640577 direct    1625046522112 09:48:42  94.2micros   127.0.0.1 ft-elasticsearch-8554868c8d-qzrck

GET _tasks/LrCxElELQGqRNI4tok60ug:234624650

{
  "error" : {
    "root_cause" : [
      {
        "type" : "resource_not_found_exception",
        "reason" : "task [LrCxElELQGqRNI4tok60ug:234624650] isn\'t running and hasn\'t stored its results"
      }
    ],
    "type" : "resource_not_found_exception",
    "reason" : "task [LrCxElELQGqRNI4tok60ug:234624650] isn\'t running and hasn\'t stored its results"
  },
  "status" : 404
}

GET _cat/indices/wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent*?v

health status index                                                 uuid                   pri rep docs.count docs.deleted store.size pri.store.size
yellow open   wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000010 EC9xHq4WQlGwBRkmZwIt2w   1   1    2731837            0      2.2gb          2.2gb
yellow open   wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000009 EfXMlKq-Q8288Twk9KJC_g   1   1    3407045            0      2.7gb          2.7gb
yellow open   wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000008 2W_fORU5RA6yUpDBm8crlQ   1   1      14046            0     13.1mb         13.1mb

GET wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000009/_count

{
  "count" : 0,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  }
}

GET wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000009/_search

{
  "took" : 0,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 0,
      "relation" : "eq"
    },
    "max_score" : null,
    "hits" : [ ]
  }
}

GET wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000009/_ilm/explain


{
  "indices" : {
    "wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000009" : {
      "index" : "wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000009",
      "managed" : true,
      "policy" : "es_rp2",
      "lifecycle_date_millis" : 1624954353191,
      "age" : "1.07d",
      "phase" : "hot",
      "phase_time_millis" : 1625033515964,
      "action" : "rollover",
      "action_time_millis" : 1624954501032,
      "step" : "ERROR",
      "step_time_millis" : 1625033689886,
      "failed_step" : "check-rollover-ready",
      "step_info" : {
        "type" : "illegal_argument_exception",
        "reason" : "index [wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000009] is not the write index for alias [wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent]",
        "stack_trace" : """
java.lang.IllegalArgumentException: index [wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000009] is not the write index for alias [wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent]
    at org.elasticsearch.xpack.core.ilm.WaitForRolloverReadyStep.evaluateCondition(WaitForRolloverReadyStep.java:100)
    at org.elasticsearch.xpack.ilm.IndexLifecycleRunner.runPeriodicStep(IndexLifecycleRunner.java:133)
    at org.elasticsearch.xpack.ilm.IndexLifecycleService.triggerPolicies(IndexLifecycleService.java:274)
    at org.elasticsearch.xpack.ilm.IndexLifecycleService.triggered(IndexLifecycleService.java:213)
    at org.elasticsearch.xpack.core.scheduler.SchedulerEngine.notifyListeners(SchedulerEngine.java:175)
    at org.elasticsearch.xpack.core.scheduler.SchedulerEngine$ActiveSchedule.run(SchedulerEngine.java:203)
    at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
    at java.base/java.lang.Thread.run(Thread.java:830)

"""
      },
      "phase_execution" : {
        "policy" : "es_rp2",
        "phase_definition" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_size" : "30gb",
              "max_age" : "14d"
            }
          }
        },
        "version" : 12,
        "modified_date_in_millis" : 1624864407438
      }
    }
  }
}

PUT wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000009/_settings
{
  "index.lifecycle.indexing_complete": "true"
}

POST wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000009/_ilm/retry

{
  "indices" : {
    "wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000009" : {
      "index" : "wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000009",
      "managed" : true,
      "policy" : "es_rp2",
      "lifecycle_date_millis" : 1624954353191,
      "age" : "1.1d",
      "phase" : "hot",
      "phase_time_millis" : 1625049429874,
      "action" : "rollover",
      "action_time_millis" : 1624954501032,
      "step" : "check-rollover-ready",
      "step_time_millis" : 1625049429874,
      "phase_execution" : {
        "policy" : "es_rp2",
        "phase_definition" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_size" : "30gb",
              "max_age" : "14d"
            }
          }
        },
        "version" : 12,
        "modified_date_in_millis" : 1624864407438
      }
    }
  }
}

GET wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000009/_ilm/explain?human

{
  "indices" : {
    "wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000009" : {
      "index" : "wksp_2dc431d6693711eb8ff97aeee04b54af_keyevent-000009",
      "managed" : true,
      "policy" : "es_rp2",
      "lifecycle_date" : "2021-06-30T10:44:37.083Z",
      "lifecycle_date_millis" : 1625049877083,
      "age" : "2.48m",
      "phase" : "hot",
      "phase_time" : "2021-06-30T10:37:09.874Z",
      "phase_time_millis" : 1625049429874,
      "action" : "complete",
      "action_time" : "2021-06-30T10:44:39.198Z",
      "action_time_millis" : 1625049879198,
      "step" : "complete",
      "step_time" : "2021-06-30T10:44:39.198Z",
      "step_time_millis" : 1625049879198,
      "phase_execution" : {
        "policy" : "es_rp2",
        "phase_definition" : {
          "min_age" : "0ms",
          "actions" : {
            "rollover" : {
              "max_size" : "30gb",
              "max_age" : "14d"
            }
          }
        },
        "version" : 12,
        "modified_date" : "2021-06-28T07:13:27.438Z",
        "modified_date_in_millis" : 1624864407438
      }
    }
  }
}

indices.lifecycle.poll_interva

生命周期策略检查间隔

三、示例

3.1 创建生命周期策略

# 创建一个生命周期管理策略
PUT _ilm/policy/yztest_policy
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_size": "50gb",
            "max_age": "30d",
            "max_docs": 10
          },
          "set_priority": {
            "priority": 100
          }
        }
      },
      "delete": {
        "min_age": "90d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}