zoukankan      html  css  js  c++  java
  • linux内核源码阅读之facebook硬盘加速flashcache之六


    其实到目前为止,如果对读流程已经能轻松地看懂了,那么写流程不需要太多脑细胞。我觉得再写下去没有太大的必要了,后面想想为了保持flashcache完整性,还是写出来吧。接着到写流程:
    1530static void
    1531flashcache_write(struct cache_c *dmc, struct bio *bio)
    1532{
    1533     int index;
    1534     int res;
    1535     struct cacheblock *cacheblk;
    1536     int queued;
    1537    
    1538     spin_lock_irq(&dmc->cache_spin_lock);
    1539     res = flashcache_lookup(dmc, bio, &index);
    1540     /*
    1541     * If cache hit and !BUSY, simply redirty page.
    1542     * If cache hit and BUSY, must wait for IO in prog to complete.
    1543     * If cache miss and found a block to recycle, we need to
    1544     * (a) invalidate any partial hits,
    1545     * (b) write to cache.
    1546     */
    1547     if (res != -1) {
    1548          /* Cache Hit */
    1549          cacheblk = &dmc->cache[index];         
    1550          if ((cacheblk->cache_state & VALID) &&
    1551              (cacheblk->dbn == bio->bi_sector)) {
    1552               /* Cache Hit */
    1553               flashcache_write_hit(dmc, bio, index);
    1554          } else {
    1555               /* Cache Miss, found block to recycle */
    1556               flashcache_write_miss(dmc, bio, index);
    1557          }
    1558          return;
    1559     }
    1560     /*
    1561     * No room in the set. We cannot write to the cache and have to
    1562     * send the request to disk. Before we do that, we must check
    1563     * for potential invalidations !
    1564     */
    1565     queued = flashcache_inval_blocks(dmc, bio);
    1566     spin_unlock_irq(&dmc->cache_spin_lock);
    1567     if (queued) {
    1568          if (unlikely(queued < 0))
    1569               flashcache_bio_endio(bio, -EIO);
    1570          return;
    1571     }
    1572     /* Start uncached IO */
    1573     flashcache_start_uncached_io(dmc, bio);
    1574     flashcache_clean_set(dmc, hash_block(dmc, bio->bi_sector));
    1575}

    第1539行查找是否命中,这里有几种情况:
    1)命中且cache空闲,直接写cache块并设置DIRTY标志
    2)命中且cache忙,等待上一个请求完成
    3)不命中并且找到可用的cache块,invalid有交集的cache块,然后再写到cache
    4)没有可用cache块,invalid有次的cache块,写到磁盘
    第4种情况在第1573行直接写到磁盘,最后调用的还是dm_io_async_bvec。
    再看第1种情况,进入到命中处理分支:
    1468static void
    1469flashcache_write_hit(struct cache_c *dmc, struct bio *bio, int index)
    1470{
    1471     struct cacheblock *cacheblk;
    1472     struct pending_job *pjob;
    1473     struct kcached_job *job;
    1474
    1475     cacheblk = &dmc->cache[index];
    1476     if (!(cacheblk->cache_state & BLOCK_IO_INPROG) && (cacheblk->head == NULL)) {
    1477          if (cacheblk->cache_state & DIRTY)
    1478               dmc->dirty_write_hits++;
    1479          dmc->write_hits++;
    1480          cacheblk->cache_state |= CACHEWRITEINPROG;
    1481          spin_unlock_irq(&dmc->cache_spin_lock);
    1482          job = new_kcached_job(dmc, bio, index);
    1483          if (unlikely(sysctl_flashcache_error_inject & WRITE_HIT_JOB_ALLOC_FAIL)) {
    1484               if (job)
    1485                    flashcache_free_cache_job(job);
    1486               job = NULL;
    1487               sysctl_flashcache_error_inject &= ~WRITE_HIT_JOB_ALLOC_FAIL;
    1488          }
    1489          if (unlikely(job == NULL)) {
    1490               /* 
    1491               * We have a write hit, and can't allocate a job.
    1492               * Since we dropped the spinlock, we have to drain any 
    1493               * pending jobs.
    1494               */
    1495               DMERR("flashcache: Write (hit) failed ! Can't allocate memory for cache IO, block %lu", 
    1496                     cacheblk->dbn);
    1497               flashcache_bio_endio(bio, -EIO);
    1498               spin_lock_irq(&dmc->cache_spin_lock);
    1499               flashcache_free_pending_jobs(dmc, cacheblk, -EIO);
    1500               cacheblk->cache_state &= ~(BLOCK_IO_INPROG);
    1501               spin_unlock_irq(&dmc->cache_spin_lock);
    1502          } else {
    1503               job->action = WRITECACHE; /* Write data to the source device */
    1504               DPRINTK("Queue job for %llu", bio->bi_sector);
    1505               atomic_inc(&dmc->nr_jobs);
    1506               dmc->ssd_writes++;
    1507               dm_io_async_bvec(1, &job->cache, WRITE, 
    1508                         bio->bi_io_vec + bio->bi_idx,
    1509                         flashcache_io_callback, job);
    1510               flashcache_unplug_device(dmc->cache_dev->bdev);
    1511               flashcache_clean_set(dmc, index / dmc->assoc);
    1512          }
    1513     } else {
    1514          pjob = flashcache_alloc_pending_job(dmc);
    1515          if (unlikely(sysctl_flashcache_error_inject & WRITE_HIT_PENDING_JOB_ALLOC_FAIL)) {
    1516               if (pjob) {
    1517                    flashcache_free_pending_job(pjob);
    1518                    pjob = NULL;
    1519               }
    1520               sysctl_flashcache_error_inject &= ~WRITE_HIT_PENDING_JOB_ALLOC_FAIL;
    1521          }
    1522          if (unlikely(pjob == NULL))
    1523               flashcache_bio_endio(bio, -EIO);
    1524          else
    1525               flashcache_enq_pending(dmc, bio, index, WRITECACHE, pjob);
    1526          spin_unlock_irq(&dmc->cache_spin_lock);
    1527     }
    1528}

    在1475行获得cache块,在1476行判断是否空闲,在有IO处理或者有pending_job挂着的时候都视为忙。如果cache块空闲,则进入if分支,接下来又是套路了,创建kcached_job,成功的话就在1507行下发写请求。然后接着看写返回时做了哪些处理?进入写回调函数之前,要记住这里设置了两个标志,一个是1480行cache块的CACHEWRITEINPROG,另一个是1503行kcached_job的WRITECACHE,带着这两个标志进入到写回调函数flashcache_io_callback,并直接找到需要的地方:
    188     case WRITECACHE:
    189          DPRINTK("flashcache_io_callback: WRITECACHE %d",
    190               index);
    191          spin_lock_irqsave(&dmc->cache_spin_lock, flags);
    192          if (unlikely(sysctl_flashcache_error_inject & WRITECACHE_ERROR)) {
    193               job->error = error = -EIO;
    194               sysctl_flashcache_error_inject &= ~WRITECACHE_ERROR;
    195          }
    196          VERIFY(cacheblk->cache_state & CACHEWRITEINPROG);
    197          if (likely(error == 0)) {
    198#ifdef FLASHCACHE_DO_CHECKSUMS
    199               dmc->checksum_store++;
    200               spin_unlock_irqrestore(&dmc->cache_spin_lock, flags);
    201               flashcache_store_checksum(job);
    202               /* 
    203               * We need to update the metadata on a DIRTY->DIRTY as well 
    204               * since we save the checksums.
    205               */
    206               push_md_io(job);
    207               schedule_work(&_kcached_wq);
    208               return;
    209#else
    210               spin_unlock_irqrestore(&dmc->cache_spin_lock, flags);
    211               /* Only do cache metadata update on a non-DIRTY->DIRTY transition */
    212               if ((cacheblk->cache_state & DIRTY) == 0) {
    213                    push_md_io(job);
    214                    schedule_work(&_kcached_wq);
    215                    return;
    216               }
    217#endif
    218          } else {
    219               dmc->ssd_write_errors++;               
    220               spin_unlock_irqrestore(&dmc->cache_spin_lock, flags);
    221          }
    222          flashcache_bio_endio(bio, error);
    223          break;

    写到缓存成功的话,暂不管cache块的校验值,会来到210行,判断原来的cache块是否为脏,如果为脏那就什么事情都不用做了。因为如果cache块本来就是脏,那新来的IO可以直接覆盖到cache块上去。反之如果原来cache块是干净的,那么这个时候要把cache块已经变脏记录到SSD上,于是进入了第213行开始写cache块管理信息。到了这里似乎cache块已经写到缓存中,IO可以返回了,但是到了第215行为什么直接return呢?这里涉及到数据一致性的问题。其实cache块管理结构没有写到缓存中,这个写请求不能算完成。试想如果在这里调用了第222行flashcache_bio_endio把IO返回了,会有什么样的后果?其实大多数情况下是没有什么问题的,但如果在这个时候系统掉电或者宕机了,这时候缓存中记录的cache块状态是干净的,但又已经跟上层返回说IO已经写成功了,那么最后这一次写的数据就丢失了。当然对于大部分用户来说,这一点数据算什么?但对于像银行这样的系统,当你把辛苦了十年的积蓄存到自动取款机,这时自动取款机告诉你存成功了,但不幸的是后台刚好发生了我们上面描述的问题。结果你再查的时候没有你刚才存进去的钱,但你的钱确确实实被取款机收进去了,这时你会有怎样的感受?这里只是举个数据一致性在某些应用中是非常重要的,当然现实中绝大数银行是不会有这样的问题,银行可以有日志查出来,系统也有热备,也是带UPS保护的。
    如果原来的cache块为脏的情况就以第222行flashcache_bio_endio结束了。
    如果不为脏,那么调用213行将cache块管理结构写到缓存。
    272void
    273push_md_io(struct kcached_job *job)
    274{
    275     push(&_md_io_jobs, job);     
    276}

    这里只是简单放到队列中,具体处理的是第214行唤醒的工作队列。该工作队列对应的处理函数是:
    303     process_jobs(&_md_io_jobs, flashcache_md_write);

    这个函数怎么这么面熟呢?因为在第一小节里已经介绍过了:
    这里小结一下写命中并且原cache块为干净的数据流程:
    1)写命中调用dm_io_async_bvec写缓存
    2)写缓存完成回调函数flashcache_io_callback,判断原cache块为干净,需要写cache块管理结构
    3)由工作队列_kcached_wq调用flashcache_md_write写cache块管理结构,最终由flashcache_md_write_kickoff调用dm_io_async_bvec将cache块管理结构写到缓存
    4)写缓存完成之后调用flashcache_md_write_callback
    5)由工作队列_kcached_wq调用flashcache_md_write_done处理
    6)在flashcache_md_write_done中判断job类型为WRITECACHE,最后调用flashcache_bio_endio返回
    至此,这个IO才完成使命。
    接下来讲第3种情况,这种情况就非常简单了。
    1411static void
    1412flashcache_write_miss(struct cache_c *dmc, struct bio *bio, int index)
    1413{
    1414     struct cacheblock *cacheblk;
    1415     struct kcached_job *job;
    1416     int queued;
    1417
    1418     cacheblk = &dmc->cache[index];
    1419     queued = flashcache_inval_blocks(dmc, bio);
    1420     if (queued) {
    1421          if (unlikely(queued < 0))
    1422               flashcache_bio_endio(bio, -EIO);
    1423          spin_unlock_irq(&dmc->cache_spin_lock);
    1424          return;
    1425     }
    1426     if (cacheblk->cache_state & VALID)
    1427          dmc->wr_replace++;
    1428     else
    1429          dmc->cached_blocks++;
    1430     cacheblk->cache_state = VALID | CACHEWRITEINPROG;
    1431     cacheblk->dbn = bio->bi_sector;
    1432     spin_unlock_irq(&dmc->cache_spin_lock);
    1433     job = new_kcached_job(dmc, bio, index);
    1434     if (unlikely(sysctl_flashcache_error_inject & WRITE_MISS_JOB_ALLOC_FAIL)) {
    1435          if (job)
    1436               flashcache_free_cache_job(job);
    1437          job = NULL;
    1438          sysctl_flashcache_error_inject &= ~WRITE_MISS_JOB_ALLOC_FAIL;
    1439     }
    1440     if (unlikely(job == NULL)) {
    1441          /* 
    1442          * We have a write miss, and can't allocate a job.
    1443          * Since we dropped the spinlock, we have to drain any 
    1444          * pending jobs.
    1445          */
    1446          DMERR("flashcache: Write (miss) failed ! Can't allocate memory for cache IO, block %lu", 
    1447                cacheblk->dbn);
    1448          flashcache_bio_endio(bio, -EIO);
    1449          spin_lock_irq(&dmc->cache_spin_lock);
    1450          dmc->cached_blocks--;
    1451          cacheblk->cache_state &= ~VALID;
    1452          cacheblk->cache_state |= INVALID;
    1453          flashcache_free_pending_jobs(dmc, cacheblk, -EIO);
    1454          cacheblk->cache_state &= ~(BLOCK_IO_INPROG);
    1455          spin_unlock_irq(&dmc->cache_spin_lock);
    1456     } else {
    1457          job->action = WRITECACHE; 
    1458          atomic_inc(&dmc->nr_jobs);
    1459          dmc->ssd_writes++;
    1460          dm_io_async_bvec(1, &job->cache, WRITE, 
    1461                    bio->bi_io_vec + bio->bi_idx,
    1462                    flashcache_io_callback, job);
    1463          flashcache_unplug_device(dmc->cache_dev->bdev);
    1464          flashcache_clean_set(dmc, index / dmc->assoc);
    1465     }
    1466}

    大多数函数都已经是老朋友了。第1430行cache块设置了VALID标志,表示在有效数据,第1431行设置cache块对应的磁盘的bi_sector扇区。接着到第1460行下发写缓存请求,写缓存的情况与写命中的一样就不再继续跟进了。
    下一节讲缓存超水位线写回磁盘。
  • 相关阅读:
    hadoop用到的shell脚本
    hadoop搭建完全分布式集群
    hadoop搭建伪分布式集群
    ARM Compute Library编译安装
    OpenBlas交叉编译安装
    正则表达式去除所有标签html标签
    xstream实现xml字符串与对象直接的转换
    spring-boot中使用mybatis-plus代码生成器让你轻松的完成单表的CURD
    开源、免费软件和网站分享
    2020春软件工程助教工作期末总结
  • 原文地址:https://www.cnblogs.com/pangblog/p/3329090.html
Copyright © 2011-2022 走看看