md.c-记录

红黑树的操作：

/* 在rb中插入一个节点，成功返回NULL，否则返回error
 * Insert a new node into the rbtree.  This returns NULL on success, or the
 * existing node on error.
 */
#define rb_insert(root, new, member, compar)                
({                                    
    struct rb_node **__n = &(root)->rb_node, *__parent = NULL;    
    typeof(new) __old = NULL, __data;                
                                    
    while (*__n) {                            
        __data = rb_entry(*__n, typeof(*new), member);        
        int __cmp = compar(new, __data);            
                                    
        __parent = *__n;                    
        if (__cmp < 0)                        
            __n = &((*__n)->rb_left);            
        else if (__cmp > 0)                    
            __n = &((*__n)->rb_right);            
        else {                            
            __old = __data;                    
            break;                        
        }                            
    }                                
                                    
    if (__old == NULL) {                        
        /* Add new node and rebalance tree. */            
        rb_link_node(&((new)->member), __parent, __n);        
        rb_insert_color(&((new)->member), root);        
    }                                
                                    
    __old;                            
})

在rbtree中查找一个节点：

/*
 * Search for a value in the rbtree.  When the key is not found in the rbtree,
 * this returns the next greater node. Note, if key > greatest node, we'll
 * return first node.
 *
 * For an empty tree, we return NULL.
 */
#define rb_nsearch(root, key, member, compar)                           
({                                                                      
        struct rb_node *__n = (root)->rb_node;                          
        typeof(key) __ret = NULL, __data;                               
                                                                        
        while (__n) {                                                   
                __data = rb_entry(__n, typeof(*key), member);           
                int __cmp = compar(key, __data);                        
                                                                        
                if (__cmp < 0) {                                        
                        __ret = __data;                                 
                        __n = __n->rb_left;                             
                } else if (__cmp > 0)                                   
                        __n = __n->rb_right;                            
                else {                                                  
                        __ret = __data;                                 
                        break;                                          
                }                                                       
        }                                                               
        if (!__ret && !RB_EMPTY_ROOT(root))                             
                __ret = rb_entry(rb_first(root), typeof(*key), member); 
        __ret;                                                          
})

创建vdisks：一个物理disk的记录就是一颗红黑子树，如果删除该disk，那么对应的是删除整棵子树

static void create_vdisks(const struct disk *disk)
{
    //计算disk路径的hash值
    uint64_t hval = sd_hash(disk->path, strlen(disk->path));
    const struct sd_node *n = &sys->this_node;
    uint64_t node_hval;
    int nr;
    // 判断当前使用的模式：Disk mode for cluster 
    if (is_cluster_diskmode(&sys->cinfo)) {
        // node的hash值
        node_hval = sd_hash(&n->nid, offsetof(typeof(n->nid), io_addr));
        hval = fnv_64a_64(node_hval, hval);
        // nr即将disk按照4G大小划分后的个数
        nr = DIV_ROUND_UP(disk->space, WEIGHT_MIN);
        if (0 == n->nid.port)
            return;
    } else
        // 按照16M空间大小划分磁盘空间后的个数
        nr = vdisk_number(disk);
        // 上述主要是获取到vdisk的个数，接下来将vdisk信息存储在rbtree中
    for (int i = 0; i < nr; i++) {
        struct vdisk *v = xmalloc(sizeof(*v));
        hval = sd_hash_next(hval);
        v->hash = hval;
        v->disk = disk;
        if (unlikely(vdisk_insert(v)))  // 在rbtree中插入vdisk，树的根在md.vroot这样的全局变量中
            panic("vdisk hash collison");
    }
}

添加一个物理磁盘：

 1 /* We don't need lock at init stage */
 2 bool md_add_disk(const char *path, bool purge)
 3 {
 4     struct disk *new;
 5     // 是否已经存在
 6     if (path_to_disk(path)) {
 7         sd_err("duplicate path %s", path);
 8         return false;
 9     }
10     // 创建相应的文件夹目录
11     if (xmkdir(path, sd_def_dmode) < 0) {
12         sd_err("can't mkdir for %s, %m", path);
13         return false;
14     }
15 
16     new = xmalloc(sizeof(*new));
17     pstrcpy(new->path, PATH_MAX, path);
18     trim_last_slash(new->path);
19     new->space = init_path_space(new->path, purge);
20     if (!new->space) {
21         free(new);
22         return false;
23     }
24     // 创建相应的vdisks
25     create_vdisks(new);
26     rb_insert(&md.root, new, rb, disk_cmp);
27     md.space += new->space;
28     md.nr_disks++;
29 
30     sd_info("%s, vdisk nr %d, total disk %d", new->path, vdisk_number(new),md.nr_disks);
31     return true;
32 }

与之相反的是删除vdisk

static inline void vdisk_free(struct vdisk *v)
{ 
    // 删除树的结构
    rb_erase(&v->rb, &md.vroot);
    free(v);
}

static void remove_vdisks(const struct disk *disk)
{
    uint64_t hval = sd_hash(disk->path, strlen(disk->path));
    const struct sd_node *n = &sys->this_node;
    uint64_t node_hval;
    int nr;

    if (is_cluster_diskmode(&sys->cinfo)) {
        node_hval = sd_hash(&n->nid, offsetof(typeof(n->nid), io_addr));
        hval = fnv_64a_64(node_hval, hval);
        nr = DIV_ROUND_UP(disk->space, WEIGHT_MIN);
    } else
        nr = vdisk_number(disk);

    for (int i = 0; i < nr; i++) {
        struct vdisk *v;

        hval = sd_hash_next(hval);
        v = hval_to_vdisk(hval);
        sd_assert(v->hash == hval);
        // 逐个释放vdisk
        vdisk_free(v);
    }
}

删除物理disk操作：

static void remove_vdisks(const struct disk *disk)
{
    uint64_t hval = sd_hash(disk->path, strlen(disk->path));
    const struct sd_node *n = &sys->this_node;
    uint64_t node_hval;
    int nr;

    if (is_cluster_diskmode(&sys->cinfo)) {
        node_hval = sd_hash(&n->nid, offsetof(typeof(n->nid), io_addr));
        hval = fnv_64a_64(node_hval, hval);
        nr = DIV_ROUND_UP(disk->space, WEIGHT_MIN);
    } else
        nr = vdisk_number(disk);

    for (int i = 0; i < nr; i++) {
        struct vdisk *v;

        hval = sd_hash_next(hval);
        v = hval_to_vdisk(hval);
        sd_assert(v->hash == hval);

        vdisk_free(v);
    }
}


static inline void md_remove_disk(struct disk *disk)
{
    sd_info("%s from multi-disk array", disk->path);
    rb_erase(&disk->rb, &md.root);
    md.nr_disks--;
    remove_vdisks(disk);
    free(disk);
}

static inline void md_del_disk(const char *path)
{
    struct disk *disk = path_to_disk(path);

    if (!disk) {
        sd_err("invalid path %s", path);
        return;
    }
    md_remove_disk(disk);
}

更新nodes disks，这里仅在配置为disk_vnodes模式才会定义：

#ifdef HAVE_DISKVNODES
void update_node_disks(void)
{
    const struct disk *disk;
    int i = 0;
    bool rb_empty = false;

    if (!sys)
        return;

    memset(sys->this_node.disks, 0, sizeof(struct disk_info) * DISK_MAX);
    sd_read_lock(&md.lock);
    rb_for_each_entry(disk, &md.root, rb) {
        sys->this_node.disks[i].disk_id =
            sd_hash(disk->path, strlen(disk->path));
        sys->this_node.disks[i].disk_space = disk->space;
        i++;
    }
    sd_rw_unlock(&md.lock);

    if (RB_EMPTY_ROOT(&md.vroot))
        rb_empty = true;
    sd_write_lock(&md.lock);
    rb_for_each_entry(disk, &md.root, rb) {
        if (!rb_empty)
            remove_vdisks(disk);
        create_vdisks(disk);
    }
    sd_rw_unlock(&md.lock);
}
#else
void update_node_disks(void)
{
}
#endif

本地磁盘插拔操作：

static int do_plug_unplug(char *disks, bool plug)
{
    const char *path;
    int old_nr, new_nr, ret = SD_RES_UNKNOWN;

    sd_write_lock(&md.lock);
    old_nr = md.nr_disks;
    path = strtok(disks, ",");
    do {
        if (plug) {
            if (!md_add_disk(path, true))
                sd_err("failed to add %s", path);
        } else {
            md_del_disk(path);
        }
    } while ((path = strtok(NULL, ",")));
    new_nr = md.nr_disks;

    /* If no disks change, bail out */
    if (old_nr == new_nr)
        goto out;

    ret = SD_RES_SUCCESS;
out:
    sd_rw_unlock(&md.lock);

    if (ret == SD_RES_SUCCESS) {
        if (new_nr > 0) {
            update_node_disks();
            kick_recover();
        } else {
            sd_warn("no disks plugged, going down");
            leave_cluster();
            sys->cinfo.status = SD_STATUS_KILLED;
        }
    }

    return ret;
}

int md_plug_disks(char *disks)
{
    return do_plug_unplug(disks, true);
}

int md_unplug_disks(char *disks)
{
    return do_plug_unplug(disks, false);
}

 oid到vdisk的映射：

/* If v1_hash < hval <= v2_hash, then oid is resident in v2，  在rbtree中寻找其位置*/
static struct vdisk *hval_to_vdisk(uint64_t hval)
{
    struct vdisk dummy = { .hash = hval };

    return rb_nsearch(&md.vroot, &dummy, rb, vdisk_cmp);
}
/* 将oid的hash值作为入参 */
static struct vdisk *oid_to_vdisk(uint64_t oid)
{    
    return hval_to_vdisk(sd_hash_oid(oid));
}