前段时间同事讲了一下AC多模匹配的原理，就试着写了下代码，现在再看代码的时候有些地方连自己都快看不懂了，所以想做一个笔记
查找10M的日志文件，即使最后一行，用时不到0.2毫秒
能正确查找出ssss串中的s,ss,sss,ssss个数和位置，notepad无法实现
1、构建树
构建模式串(需要搜索的串)为{"hee","he", "h","she", "his" ,"her","hers"}的树。注：下划线序号表示层高
构建过程也是查找的过程，举例讲解构建：hee过程，其他类似
构建hee过程：
1、构建h节点：输入串为h,以根节点作为起始节点SNode，查找SNode下是否存在值为h,层高为strlen(h)=1的节点，不存在，插入
2、构建第一个e节点：输入串为he,以根节点作为起始节点SNode，查找SNode下是否存在值为h,层高为strlen(h)=1的节点，存在，以h节点作为SNode,查找是否存值为e,层高为strlen(he)的节点，不存在，插入
3、构建第二个e节点：输入串为hee,以根节点作为起始节点SNode，查找SNode下是否存在值为h,层高为strlen(h)=1的节点，存在，以h节点作为SNode,查找是否存值为e,层高为strlen(he)的节点，存在，以e节点作为SNode节点，查找是否存值为e,层高为strlen(hee)的节点,不存在，插入。构建字符e为构建串的最后一个字符时，该节点为输出节点
构建一个串中某个字符的节点时，会将该字符的前面所有字符+该字符作为一个串输入，搜索树并插入。因为每个字符必须要知道前面的路径
2、构建失败节点
- 根节点的失败节点为根节点，第一层节点的失败节点为根节点
- 某个节点的失败节点为：以该节点的父节点的失败节点为起点查找该节点，如果能找到，找到的节点为失败节点，不能找到，则失败节点为根节点
已知： f(root) = root;f(h_1)=root;f(s_1)=root
解： f(e_2) = g(f(h_1),e) = g(root,e) = root
f(e_3) = g(f(e_2),e) = g(root,e) = root
f(r_3) = g(f(e_2),r) = g(root,r) = root
f(s_4) = g(f(r_3),s) = g(root,s) = s_1
f(i_2) = g(f(h_1),i) = g(root,i) = root
f(s_3) = g(f(i_2),s) = g(root,s) = s_1
f(h_2) = g(f(s_1),h) = g(root,h) = h_1
f(e_3) = g(f(h_2),e)=g(h_1,e)=e_2
3、查找"ushers"中的模式串
1、遍历"ushers",从根节点作为起始节点SNode,查找字符为u的节点，查找失败，以SNode的失败节点为起始节点查找下一个字符s
2、以SNode作为起始节点，查找字符为s的节点，查找成功，判断节点是否为匹配节点，输出匹配值，以s节点作为起始节点查找h节点，直到遍历完"ushers"
大家可以手动试一下构建模式串为{"s","ss","sss","ssss"}的树，构建每个节点的失败节点,并在内容"ssss"中查找模式串
文字只讲了大致流程，还有很多细节没有描述出来，语言水平有限啊，细节大家看代码吧...
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
AC.h
#ifndef AC_H
#define AC_H
#define MAX_CHILD_LEN 128
#define MAX_OUTPUT 128
#include <string.h>
#include <stdio.h>
class Node
{
public:
Node(char ele){
element = ele;
parent = NULL;
failNode = NULL;
isMatchNode=false;
memset(nodeList,0,MAX_CHILD_LEN);
childNum=0;
high = 0;
memset(outPut,0,MAX_OUTPUT);
outPutNum = 0;
}
~Node(){
printf("node release,high:%d value:%c\\n",high,element);
for(int i=0;i<outPutNum;i++){
char * ele = outPut[i];
printf("release output str:%s\\n",ele);
delete ele;
ele = NULL;
}
}
char element;
Node * parent;
Node * nodeList[MAX_CHILD_LEN];
Node * failNode;
bool isMatchNode;
int childNum;
int high;
char * outPut[MAX_OUTPUT];
int outPutNum;
};
typedef void(*Func)(char * matchedStr,int post);
class AC
{
public:
AC() {root = new Node(NULL);root->failNode = root;}
~AC(){
delete root;
root = NULL;
}
void initTree(char * patterns[],int patLen){
for(int i=0;i<patLen;i++){
char * element = patterns[i];
int eleLen = strlen(element);
for(int j=0;j<eleLen;j++){
bool isMatch = false;
if(j == eleLen-1)
isMatch = true;
char * p = new char[j+2];
memset(p,‘\\0‘,j+2);
strncpy(p,element,j+1);
insert(p,isMatch);
}
}
}
void buildFailNode(){
traceAllNodes(root);
}
void match(char * srcTxt,int txtLen,Func f){
Node * startNode = root;
for(int i=0;i<txtLen;i++){
char e = srcTxt[i];
bool isOk = false;
for(int j=0;j<startNode->childNum;j++){
Node * node = startNode->nodeList[j];
if(node->element == e){
isOk = true;
startNode = node;
Node * failNode = node->failNode;
while(failNode!=root){
if(failNode->isMatchNode){
for(int k=0;k<failNode->outPutNum;k++)
f(failNode->outPut[k],i);
}
failNode = failNode->failNode;
}
if(node->isMatchNode){
for(int k=0;k<node->outPutNum;k++)
f(node->outPut[k],i);
}
break;
}
}
if(!isOk){
startNode = startNode->failNode;
if(startNode!=root)
i--;
}
}
}
void deleteTree(){
printf("delete tree--------------------------------\\n");
traceDelNodes(root);
}
void printACTree(){
printf("tree structure-----------------------------\\n");
printf("high value match failNode childNum children outPutStr \\n");
tracePrintNodes(root);
}
private:
void insert(char * ele,bool isMatch){
int eleLen = strlen(ele);
//搜索ele最后一个元素节点是否存在，不存在则返回父节点
int startH = 1;
Node * pnode = NULL;
if(!search(root,startH,ele,pnode)){
Node * cnode = new Node(ele[eleLen-1]);
cnode->high = eleLen;
cnode->parent = pnode;
cnode->isMatchNode = isMatch;
pnode->nodeList[pnode->childNum]=cnode;
pnode->childNum++;
if(isMatch)
cnode->outPut[cnode->outPutNum++]=ele;
}
}
bool search(Node * pnode,int & index,char * ele,Node * &retNode){
for(int i=0;i<pnode->childNum;i++){
Node * node = pnode->nodeList[i];
if(node->element == ele[index-1]){
if(index == strlen(ele))
return true;
index++;
return search(node,index,ele,retNode);
}
}
retNode = pnode;
return false;
}
void initFailNode(Node * node){
if(node->high == 1){
node->failNode = root;
}else{
//以父节点的失败函数作为起点，node的element作为触发边得到node的失败函数
Node * failNode = NULL;
searchFailNode(node->parent,node,failNode);
node->failNode = failNode;
}
}
void traceAllNodes(Node * node){
for(int i=0;i<node->childNum;i++){
Node * cnode = node->nodeList[i];
initFailNode(cnode);
traceAllNodes(cnode);
}
}
void searchFailNode(Node * pnode,Node * cnode,Node * & retNode){
Node * failNode = pnode->failNode;
for(int i=0;i<failNode->childNum;i++){
Node * node = failNode->nodeList[i];
if(node->element == cnode->element){
retNode = node;
return;
}
}
//循环已经走完，说明没找到节点，如果已经搜索了根节点没找到，则返回根节点作为失败节点，否则继续搜索
if(pnode->failNode == root)
retNode = root;
else
searchFailNode(pnode->parent,pnode,retNode);
}
void tracePrintNodes(Node *node){
for(int i=0;i<node->childNum;i++){
Node * cnode = node->nodeList[i];
printf(" %d %c %d %d %d ",cnode->high,cnode->element,cnode->isMatchNode,cnode->failNode->high,cnode->childNum);
for(int j=0;j<cnode->childNum;j++){
Node * lnode = cnode->nodeList[j];
printf("%c ",lnode->element);
}
printf(" ");
for(int j=0;j<cnode->outPutNum;j++)
printf("%s ",cnode->outPut[j]);
printf("\\n");
tracePrintNodes(cnode);
}
}
void traceDelNodes(Node * node){
for(int i=0;i<node->childNum;i++){
Node * cnode = node->nodeList[i];
traceDelNodes(cnode);
}
if(node!=root){
delete node;
node = NULL;
}
}
private:
Node * root;
};
#endif // AC_H
-----------------------------------------------------------------------------------
main.cpp
#include "AC.h"
#include <time.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <unistd.h>
#include <map>
#define MAX_FILE_CONTENT 1024*1024*20
void currentTime(char * timeOutPut,int timeLen){
time_t t = time(NULL);
struct tm * timeinfo = localtime(&t);
strftime (timeOutPut,timeLen,"%Y-%m-%d %H:%M:%S:",timeinfo);
struct timeval tval;
gettimeofday(&tval,NULL);
sprintf(timeOutPut+strlen(timeOutPut),"%d",tval.tv_usec/1000);
}
//回调函数在外面进行统计，找到一个回调一次
void findCallBack(char * matchedStr,int startPos){
char curTime[32]={0};
currentTime(curTime,sizeof(curTime));
printf("end time:%s matchedStr:%s matchedPos:%d\\n",curTime,matchedStr,startPos);
}
bool readFile(char * fileName,char * fileContent){
int fd = open(fileName,O_RDONLY);
if(-1 == fd){
printf("open file error:%d\\n",errno);
return false;
}
int len = read(fd,fileContent,MAX_FILE_CONTENT);
if(-1 == len){
printf("read file error:%d\\n",errno);
close(fd);
return false;
}
close(fd);
return true;
}
void ac_func(char ** pattern,int patLen,char *txt,int txtLen){
AC ac;
ac.initTree(pattern,patLen);
ac.buildFailNode();
ac.printACTree();
char curTime[32]={0};
currentTime(curTime,sizeof(curTime));
printf("searching tree-----------------------------\\n");
printf("start time of search:%s\\n",curTime);
ac.match(txt,txtLen,findCallBack);
ac.deleteTree();
}
void test1(){
char *pattern[]={"hee","he", "h","she", "his" ,"her","hers"};
char *txt = "ushers";
int patternLen = sizeof(pattern)/sizeof(char*);
int txtLen = strlen(txt);
ac_func(pattern,patternLen,txt,txtLen);
}
void test2(){
//模式串出现的顺序在文件中刚好相反，里面存在一个关键字在日志文件的最后一行
char * pattern[] = {"39_347990193541512029","39_347739859976612007","13_1002D375","2017-09-29 17:43:37:517"};
//文件内容太大，使用堆内存
char * txt = new char[MAX_FILE_CONTENT];
bzero(txt,MAX_FILE_CONTENT);
//test.log为一个10M的日志
if(!readFile("../testfile/test.log",txt))
return;
int patternLen = sizeof(pattern)/sizeof(char*);
int txtLen = strlen(txt);
ac_func(pattern,patternLen,txt,txtLen);
delete txt;
txt = NULL;
}
void test3(){
//模式串和内容出现自包含,notepad遇到这样的搜索有bug
char *pattern[]={"s", "ss","sss","ssss"};
char *txt = "ssss";
int patternLen = sizeof(pattern)/sizeof(char*);
int txtLen = strlen(txt);
ac_func(pattern,patternLen,txt,txtLen);
}
int main()
{
test1();
test2();
test3();
return 0;
}
----------------------------------------------------------------------------------------------------
运行结果部分截图
