Redis系列：数据结构Hash源码解析和HSETHGET命令
Posted QQ:359855043
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了Redis系列：数据结构Hash源码解析和HSETHGET命令相关的知识，希望对你有一定的参考价值。
2.源码解析
1.相关命令如下：
{"hset",hsetCommand,4,"wmF",0,NULL,1,1,1,0,0},
{"hsetnx",hsetnxCommand,4,"wmF",0,NULL,1,1,1,0,0},
{"hget",hgetCommand,3,"rF",0,NULL,1,1,1,0,0},
{"hmset",hmsetCommand,-4,"wm",0,NULL,1,1,1,0,0},
{"hmget",hmgetCommand,-3,"r",0,NULL,1,1,1,0,0},
{"hincrby",hincrbyCommand,4,"wmF",0,NULL,1,1,1,0,0},
{"hincrbyfloat",hincrbyfloatCommand,4,"wmF",0,NULL,1,1,1,0,0},
{"hdel",hdelCommand,-3,"wF",0,NULL,1,1,1,0,0},
{"hlen",hlenCommand,2,"rF",0,NULL,1,1,1,0,0},
{"hstrlen",hstrlenCommand,3,"rF",0,NULL,1,1,1,0,0},
{"hkeys",hkeysCommand,2,"rS",0,NULL,1,1,1,0,0},
{"hvals",hvalsCommand,2,"rS",0,NULL,1,1,1,0,0},
{"hgetall",hgetallCommand,2,"r",0,NULL,1,1,1,0,0},
{"hexists",hexistsCommand,3,"rF",0,NULL,1,1,1,0,0},
{"hscan",hscanCommand,-3,"rR",0,NULL,1,1,1,0,0},
2.ziplist数据结构
/* We use this function to receive information about a ziplist entry. * Note that this is not how the data is actually encoded, is just what we * get filled by a function in order to operate more easily. */ typedef struct zlentry { unsigned int prevrawlensize; /* Bytes used to encode the previous entry len*/ unsigned int prevrawlen; /* Previous entry len. */ unsigned int lensize; /* Bytes used to encode this entry type/len. For example strings have a 1, 2 or 5 bytes header. Integers always use a single byte.*/ unsigned int len; /* Bytes used to represent the actual entry. For strings this is just the string length while for integers it is 1, 2, 3, 4, 8 or 0 (for 4 bit immediate) depending on the number range. */ unsigned int headersize; /* prevrawlensize + lensize. */ unsigned char encoding; /* Set to ZIP_STR_* or ZIP_INT_* depending on the entry encoding. However for 4 bits immediate integers this can assume a range of values and must be range-checked. */ unsigned char *p; /* Pointer to the very start of the entry, that is, this points to prev-entry-len field. */ } zlentry;
3.hashtable数据结构
typedef struct dictEntry { void *key; union { void *val; uint64_t u64; int64_t s64; double d; } v; struct dictEntry *next; } dictEntry; typedef struct dictType { uint64_t (*hashFunction)(const void *key); void *(*keyDup)(void *privdata, const void *key); void *(*valDup)(void *privdata, const void *obj); int (*keyCompare)(void *privdata, const void *key1, const void *key2); void (*keyDestructor)(void *privdata, void *key); void (*valDestructor)(void *privdata, void *obj); } dictType; /* This is our hash table structure. Every dictionary has two of this as we * implement incremental rehashing, for the old to the new table. */ typedef struct dictht { dictEntry **table; unsigned long size; unsigned long sizemask; unsigned long used; } dictht; typedef struct dict { dictType *type; void *privdata; dictht ht[2]; long rehashidx; /* rehashing not in progress if rehashidx == -1 */ unsigned long iterators; /* number of iterators currently running */ } dict;
hset
// t_hash.c, set key field value void hsetCommand(client *c) { int update; robj *o; // 1. 查找hash的key是否存在，不存在则新建一个，然后在其上进行数据操作 if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return; // 2. 检查2-3个参数是否需要将简单版（ziplist）hash表转换为复杂的hash表，转换后的表通过 o->ptr 体现 hashTypeTryConversion(o,c->argv,2,3); // 3. 添加kv到 o 的hash表中 update = hashTypeSet(o,c->argv[2]->ptr,c->argv[3]->ptr,HASH_SET_COPY); addReply(c, update ? shared.czero : shared.cone); // 变更命令传播 signalModifiedKey(c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_HASH,"hset",c->argv[1],c->db->id); server.dirty++; } // 1. 获取db外部的key, 即整体hash数据实例 // t_hash.c robj *hashTypeLookupWriteOrCreate(client *c, robj *key) { robj *o = lookupKeyWrite(c->db,key); if (o == NULL) { // 此处创建的hashObject是以 ziplist 形式的 o = createHashObject(); dbAdd(c->db,key,o); } else { // 不是hash类型的键已存在，不可覆盖，返回错误 if (o->type != OBJ_HASH) { addReply(c,shared.wrongtypeerr); return NULL; } } return o; } // object.c, 创建hashObject, 以 ziplist 形式创建 robj *createHashObject(void) { unsigned char *zl = ziplistNew(); robj *o = createObject(OBJ_HASH, zl); o->encoding = OBJ_ENCODING_ZIPLIST; return o; } // ziplist.c static unsigned char *createList() { unsigned char *zl = ziplistNew(); zl = ziplistPush(zl, (unsigned char*)"foo", 3, ZIPLIST_TAIL); zl = ziplistPush(zl, (unsigned char*)"quux", 4, ZIPLIST_TAIL); zl = ziplistPush(zl, (unsigned char*)"hello", 5, ZIPLIST_HEAD); zl = ziplistPush(zl, (unsigned char*)"1024", 4, ZIPLIST_TAIL); return zl; } // 2. 检查参数，是否需要将 ziplist 形式的hash表转换为真正的hash表 /* Check the length of a number of objects to see if we need to convert a * ziplist to a real hash. Note that we only check string encoded objects * as their string length can be queried in constant time. */ void hashTypeTryConversion(robj *o, robj **argv, int start, int end) { int i; if (o->encoding != OBJ_ENCODING_ZIPLIST) return; for (i = start; i <= end; i++) { // 参数大于设置的 hash_max_ziplist_value (默认: 64)时，会直接将 ziplist 转换为 ht // OBJ_ENCODING_RAW, OBJ_ENCODING_EMBSTR // 循环检查参数，只要发生了一次转换就结束检查（没必要继续了） if (sdsEncodedObject(argv[i]) && sdslen(argv[i]->ptr) > server.hash_max_ziplist_value) { // 这个转换过程很有意思，我们深入看看 hashTypeConvert(o, OBJ_ENCODING_HT); break; } } } // t_hash.c, 转换编码方式 (如上, ziplist -> ht) void hashTypeConvert(robj *o, int enc) { if (o->encoding == OBJ_ENCODING_ZIPLIST) { // 此处我们只处理这种情况 hashTypeConvertZiplist(o, enc); } else if (o->encoding == OBJ_ENCODING_HT) { serverPanic("Not implemented"); } else { serverPanic("Unknown hash encoding"); } } // t_hash.c, 转换编码 ziplist 为目标 enc (实际只能是 OBJ_ENCODING_HT) void hashTypeConvertZiplist(robj *o, int enc) { serverAssert(o->encoding == OBJ_ENCODING_ZIPLIST); if (enc == OBJ_ENCODING_ZIPLIST) { /* Nothing to do... */ } else if (enc == OBJ_ENCODING_HT) { hashTypeIterator *hi; dict *dict; int ret; // 迭代器创建 hi = hashTypeInitIterator(o); // 一个hash的数据结构就是一个 dict, 从这个级别来说, hash 与 db 是一个级别的 dict = dictCreate(&hashDictType, NULL); // 依次迭代 o, 赋值到 hi->fptr, hi->vptr // 依次添加到 dict 中 while (hashTypeNext(hi) != C_ERR) { sds key, value; // 从 hi->fptr 中获取key // 从 hi->vptr 中获取value key = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_KEY); value = hashTypeCurrentObjectNewSds(hi,OBJ_HASH_VALUE); // 添加到 dict 中 ret = dictAdd(dict, key, value); if (ret != DICT_OK) { serverLogHexDump(LL_WARNING,"ziplist with dup elements dump", o->ptr,ziplistBlobLen(o->ptr)); serverPanic("Ziplist corruption detected"); } } // 释放迭代器 hashTypeReleaseIterator(hi); zfree(o->ptr); // 将变更反映到o对象上返回 o->encoding = OBJ_ENCODING_HT; o->ptr = dict; } else { serverPanic("Unknown hash encoding"); } } // 2.1. 迭代ziplist元素 // t_hash.c, 迭代器 /* Move to the next entry in the hash. Return C_OK when the next entry * could be found and C_ERR when the iterator reaches the end. */ int hashTypeNext(hashTypeIterator *hi) { if (hi->encoding == OBJ_ENCODING_ZIPLIST) { unsigned char *zl; unsigned char *fptr, *vptr; // 每次都是基于原始字符器进行计算偏移 // 迭代的是 fptr,vptr zl = hi->subject->ptr; fptr = hi->fptr; vptr = hi->vptr; // 第一次查找时使用index查找，后续则使用 fptr,vptr 进行迭代 if (fptr == NULL) { /* Initialize cursor */ serverAssert(vptr == NULL); fptr = ziplistIndex(zl, 0); } else { /* Advance cursor */ serverAssert(vptr != NULL); fptr = ziplistNext(zl, vptr); } if (fptr == NULL) return C_ERR; /* Grab pointer to the value (fptr points to the field) */ vptr = ziplistNext(zl, fptr); serverAssert(vptr != NULL); /* fptr, vptr now point to the first or next pair */ hi->fptr = fptr; hi->vptr = vptr; } else if (hi->encoding == OBJ_ENCODING_HT) { if ((hi->de = dictNext(hi->di)) == NULL) return C_ERR; } else { serverPanic("Unknown hash encoding"); } return C_OK; } // ziplist.c, 查找 index 的元素 /* Returns an offset to use for iterating with ziplistNext. When the given * index is negative, the list is traversed back to front. When the list * doesn\'t contain an element at the provided index, NULL is returned. */ unsigned char *ziplistIndex(unsigned char *zl, int index) { unsigned char *p; unsigned int prevlensize, prevlen = 0; if (index < 0) { // 小于0时，反向查找 index = (-index)-1; p = ZIPLIST_ENTRY_TAIL(zl); if (p[0] != ZIP_END) { ZIP_DECODE_PREVLEN(p, prevlensize, prevlen); while (prevlen > 0 && index--) { p -= prevlen; ZIP_DECODE_PREVLEN(p, prevlensize, prevlen); } } } else { p = ZIPLIST_ENTRY_HEAD(zl); while (p[0] != ZIP_END && index--) { p += zipRawEntryLength(p); } } // 迭代完成还没找到元素 p[0]=ZIP_END // index 超出整体ziplist大小则遍历完成后 index>0 return (p[0] == ZIP_END || index > 0) ? NULL : p; } // ziplist.c, 由 fptr,vptr 进行迭代元素 /* Return pointer to next entry in ziplist. * * zl is the pointer to the ziplist * p is the pointer to the current element * * The element after \'p\' is returned, otherwise NULL if we are at the end. */ unsigned char *ziplistNext(unsigned char *zl, unsigned char *p) { ((void) zl); /* "p" could be equal to ZIP_END, caused by ziplistDelete, * and we should return NULL. Otherwise, we should return NULL * when the *next* element is ZIP_END (there is no next entry). */ if (p[0] == ZIP_END) { return NULL; } // 当前指针偏移当前元素长度（根据ziplist协议），即到下一元素指针位置 p += zipRawEntryLength(p); if (p[0] == ZIP_END) { return NULL; } return p; } /* Return the total number of bytes used by the entry pointed to by \'p\'. */ static unsigned int zipRawEntryLength(unsigned char *p) { unsigned int prevlensize, encoding, lensize, len; ZIP_DECODE_PREVLENSIZE(p, prevlensize); ZIP_DECODE_LENGTH(p + prevlensize, encoding, lensize, len); return prevlensize + lensize + len; } // 2.2. t_hash.c, 获取 hashTypeIterator 的具体值，写入 vstr, vlen 中 /* Return the key or value at the current iterator position as a new * SDS string. */ sds hashTypeCurrentObjectNewSds(hashTypeIterator *hi, int what) { unsigned char *vstr; unsigned int vlen; long long vll; hashTypeCurrentObject(hi,what,&vstr,&vlen,&vll); if (vstr) return sdsnewlen(vstr,vlen); return sdsfromlonglong(vll); } /* Higher level function of hashTypeCurrent*() that returns the hash value * at current iterator position. * * The returned element is returned by reference in either *vstr and *vlen if * it\'s returned in string form, or stored in *vll if it\'s returned as * a number. * * If *vll is populated *vstr is set to NULL, so the caller * can always check the function return by checking the return value * type checking if vstr == NULL. */ void hashTypeCurrentObject(hashTypeIterator *hi, int what, unsigned char **vstr, unsigned int *vlen, long long *vll) { if (hi->encoding == OBJ_ENCODING_ZIPLIST) { *vstr = NULL; hashTypeCurrentFromZiplist(hi, what, vstr, vlen, vll); } else if (hi->encoding == OBJ_ENCODING_HT) { sds ele = hashTypeCurrentFromHashTable(hi, what); *vstr = (unsigned char*) ele; *vlen = sdslen(ele); } else { serverPanic("Unknown hash encoding"); } } // t_hash.c, 从ziplist中获取某个 hashTypeIterator 的具体值，结果定稿 vstr, vlen /* Get the field or value at iterator cursor, for an iterator on a hash value * encoded as a ziplist. Prototype is similar to `hashTypeGetFromZiplist`. */ void hashTypeCurrentFromZiplist(hashTypeIterator *hi, int what, unsigned char **vstr, unsigned int *vlen, long long *vll) { int ret; serverAssert(hi->encoding == OBJ_ENCODING_ZIPLIST); // OBJ_HASH_KEY 从 fptr 中获取, 否则从 vptr 中获取 if (what & OBJ_HASH_KEY) { ret = ziplistGet(hi->fptr, vstr, vlen, vll); serverAssert(ret); } else { ret = ziplistGet(hi->vptr, vstr, vlen, vll); serverAssert(ret); } } // ziplist.c, /* Get entry pointed to by \'p\' and store in either \'*sstr\' or \'sval\' depending * on the encoding of the entry. \'*sstr\' is always set to NULL to be able * to find out whether the string pointer or the integer value was set. * Return 0 if \'p\' points to the end of the ziplist, 1 otherwise. */ unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *slen, long long *sval) { zlentry entry; if (p == NULL || p[0] == ZIP_END) return 0; if (sstr) *sstr = NULL; // 按照ziplist的编码协议, 获取头部信息 zipEntry(p, &entry); if (ZIP_IS_STR(entry.encoding)) { if (sstr) { *slen = entry.len; *sstr = p+entry.headersize; } } else { if (sval) { *sval = zipLoadInteger(p+entry.headersize,entry.encoding); } } return 1; } // ziplist.c, 解析原始字符串为 zlentry /* Return a struct with all information about an entry. */ static void zipEntry(unsigned char *p, zlentry *e) { // 按照ziplist的编码协议,依次读取 prevrawlensize, prevrawlen ZIP_DECODE_PREVLEN(p, e->prevrawlensize, e->prevrawlen); // 指向下一位置偏移，按照ziplist的编码协议,依次读取 encoding, lensize, len ZIP_DECODE_LENGTH(p + e->prevrawlensize, e->encoding, e->lensize, e->len); // 除去header得到 body偏移 e->headersize = e->prevrawlensize + e->lensize; e->p = p; }
header
// ziplist.c /* Decode the length of the previous element, from the perspective of the entry * pointed to by \'ptr\'. */ #define ZIP_DECODE_PREVLEN(ptr, prevlensize, prevlen) do { \\ // 解析第1个字符为 prevlensize ZIP_DECODE_PREVLENSIZE(ptr, prevlensize); \\ if ((prevlensize) == 1) { \\ (prevlen) = (ptr)[0]; \\ } else if ((prevlensize) == 5) { \\ assert(sizeof((prevlensize)) == 4); \\ // 当ptr[0]>254时，代表内容有点大，需要使用 5个字符保存上一字符长度 memcpy(&(prevlen), ((char*)(ptr)) + 1, 4); \\ memrev32ifbe(&prevlen); \\ } \\ } while(0); /* Decode the number of bytes required to store the length of the previous * element, from the perspective of the entry pointed to by \'ptr\'. */ #define ZIP_DECODE_PREVLENSIZE(ptr, prevlensize) do { \\ if ((ptr)[0] < ZIP_BIGLEN) { \\ (prevlensize) = 1; \\ } else { \\ (prevlensize) = 5; \\ } \\ } while(0); /* Decode the length encoded in \'ptr\'. The \'encoding\' variable will hold the * entries encoding, the \'lensize\' variable will hold the number of bytes * required to encode the entries length, and the \'len\' variable will hold the * entries length. */ #define ZIP_DECODE_LENGTH(ptr, encoding, lensize, len) do { \\ // 解析第1个字符为 编码格式 &ZIP_STR_MASK=0xc0 ZIP_ENTRY_ENCODING((ptr), (encoding)); \\ if ((encoding) < ZIP_STR_MASK) { \\ // 0 << 6 =0 // 具体解析如下代码, if ((encoding) == ZIP_STR_06B) { \\ (lensize) = 1; \\ (len) = (ptr)[0] & 0x3f; \\ } // 1 << 6 =64 else if ((encoding) == ZIP_STR_14B) { \\ (lensize) = 2; \\ (len) = (((ptr)[0] & 0x3f) << 8) | (ptr)[1]; \\ } // 2 << 6 =128 else if (encoding == ZIP_STR_32B) { \\ (lensize) = 5; \\ (len) = ((ptr)[1] << 24) | \\ ((ptr)[2] << 16) | \\ ((ptr)[3] << 8) | \\ ((ptr)[4]); \\ } else { \\ assert(NULL); \\ } \\ } else { \\ // 超过 0xc0 的长度了，直接使用 1,2,3,4 表示len (lensize) = 1; \\ (len) = zipIntSize(encoding); \\ } \\ } while(0); /* Extract the encoding from the byte pointed by \'ptr\' and set it into * \'encoding\'. */ #define ZIP_ENTRY_ENCODING(ptr, encoding) do { \\ (encoding) = (ptr[0]); \\ if ((encoding) < ZIP_STR_MASK) (encoding) &= ZIP_STR_MASK; \\ } while(0) /* Different encoding/length possibilities */ #define ZIP_STR_MASK 0xc0 #define ZIP_INT_MASK 0x30 #define ZIP_STR_06B (0 << 6) // 0x00 #define ZIP_STR_14B (1 << 6) // 0x40 #define ZIP_STR_32B以上是关于Redis系列：数据结构Hash源码解析和HSETHGET命令的主要内容，如果未能解决你的问题，请参考以下文章