diff --git a/Documentation/admin-guide/device-mapper/dm-pcache.rst b/Documentation/admin-guide/device-mapper/dm-pcache.rst index 09d327ef4b14ee..2c87d542f8b48a 100644 --- a/Documentation/admin-guide/device-mapper/dm-pcache.rst +++ b/Documentation/admin-guide/device-mapper/dm-pcache.rst @@ -29,7 +29,7 @@ Constructor :: - pcache [ ] + pcache [ ] ========================= ==================================================== ``cache_dev`` Any DAX-capable block device (``/dev/pmem0``…). @@ -37,8 +37,7 @@ Constructor ``backing_dev`` The slow block device to be cached. -``cache_mode`` Optional, Only ``writeback`` is accepted at the - moment. +``cache_mode`` Optional, ``writeback``, ``writethrough``, ``writearound``, ``writeonly``. ``data_crc`` Optional, default to ``false`` diff --git a/drivers/md/dm-pcache/cache.c b/drivers/md/dm-pcache/cache.c index 4d6db733c9bdd6..7a6c6841cbb193 100644 --- a/drivers/md/dm-pcache/cache.c +++ b/drivers/md/dm-pcache/cache.c @@ -53,6 +53,13 @@ static int cache_info_init(struct pcache_cache *cache, struct pcache_cache_optio cache->info_index = ((char *)cache_info_addr - (char *)cache->cache_info_addr) / PCACHE_CACHE_INFO_SIZE; + if (opts->cache_mode != cache_mode_get(cache)) { + pcache_dev_err(pcache, "invalid option for cache_mode: %s, expected: %s", + cache_mode_str(opts->cache_mode), + cache_mode_str(cache_mode_get(cache))); + return -EINVAL; + } + return 0; } diff --git a/drivers/md/dm-pcache/cache.h b/drivers/md/dm-pcache/cache.h index 27613b56be54c2..b127faa8bda22a 100644 --- a/drivers/md/dm-pcache/cache.h +++ b/drivers/md/dm-pcache/cache.h @@ -122,6 +122,7 @@ struct pcache_cache_key { #define PCACHE_CACHE_KEY_FLAGS_EMPTY BIT(0) #define PCACHE_CACHE_KEY_FLAGS_CLEAN BIT(1) +#define PCACHE_CACHE_KEY_FLAGS_CLEAR BIT(2) struct pcache_cache_key_onmedia { __u64 off; @@ -399,6 +400,11 @@ static inline bool cache_key_clean(struct pcache_cache_key *key) return key->flags & PCACHE_CACHE_KEY_FLAGS_CLEAN; } +static inline bool cache_key_clear(struct pcache_cache_key *key) +{ + return key->flags & PCACHE_CACHE_KEY_FLAGS_CLEAR; +} + static inline void cache_pos_copy(struct pcache_cache_pos *dst, struct pcache_cache_pos *src) { memcpy(dst, src, sizeof(struct pcache_cache_pos)); @@ -476,6 +482,78 @@ static inline void cache_mode_set(struct pcache_cache *cache, u32 cache_mode) cache->cache_info.flags |= FIELD_PREP(PCACHE_CACHE_FLAGS_CACHE_MODE_MASK, cache_mode); } +static inline const char *cache_mode_str(u32 cache_mode) +{ + switch (cache_mode) { + case PCACHE_CACHE_MODE_WRITEBACK: + return "writeback"; + case PCACHE_CACHE_MODE_WRITETHROUGH: + return "writethrough"; + case PCACHE_CACHE_MODE_WRITEAROUND: + return "writearound"; + case PCACHE_CACHE_MODE_WRITEONLY: + return "writeonly"; + default: + BUG(); + } +} + +static inline bool cache_mode_need_writeback(struct pcache_cache *cache) +{ + switch (cache_mode_get(cache)) { + case PCACHE_CACHE_MODE_WRITEBACK: + case PCACHE_CACHE_MODE_WRITEONLY: + return true; + case PCACHE_CACHE_MODE_WRITETHROUGH: + case PCACHE_CACHE_MODE_WRITEAROUND: + return false; + default: + BUG(); + } +} + +static inline bool cache_mode_need_cache(struct pcache_cache *cache) +{ + switch (cache_mode_get(cache)) { + case PCACHE_CACHE_MODE_WRITEBACK: + case PCACHE_CACHE_MODE_WRITETHROUGH: + case PCACHE_CACHE_MODE_WRITEONLY: + return true; + case PCACHE_CACHE_MODE_WRITEAROUND: + return false; + default: + BUG(); + } +} + +static inline bool cache_mode_need_backing(struct pcache_cache *cache) +{ + switch (cache_mode_get(cache)) { + case PCACHE_CACHE_MODE_WRITETHROUGH: + case PCACHE_CACHE_MODE_WRITEAROUND: + return true; + case PCACHE_CACHE_MODE_WRITEBACK: + case PCACHE_CACHE_MODE_WRITEONLY: + return false; + default: + BUG(); + } +} + +static inline bool cache_mode_need_read(struct pcache_cache *cache) +{ + switch (cache_mode_get(cache)) { + case PCACHE_CACHE_MODE_WRITETHROUGH: + case PCACHE_CACHE_MODE_WRITEAROUND: + case PCACHE_CACHE_MODE_WRITEBACK: + return true; + case PCACHE_CACHE_MODE_WRITEONLY: + return false; + default: + BUG(); + } +} + /** * cache_key_data_crc - Calculates CRC for data in a cache key. * @key: Pointer to the pcache_cache_key structure. diff --git a/drivers/md/dm-pcache/cache_gc.c b/drivers/md/dm-pcache/cache_gc.c index 94f8b276a02128..94dee6c4376dbe 100644 --- a/drivers/md/dm-pcache/cache_gc.c +++ b/drivers/md/dm-pcache/cache_gc.c @@ -114,9 +114,18 @@ void pcache_cache_gc_fn(struct work_struct *work) return; /* Get new tail positions */ - mutex_lock(&cache->dirty_tail_lock); - cache_pos_copy(&dirty_tail, &cache->dirty_tail); - mutex_unlock(&cache->dirty_tail_lock); + if (cache_mode_need_writeback(cache)) { + mutex_lock(&cache->dirty_tail_lock); + cache_pos_copy(&dirty_tail, &cache->dirty_tail); + mutex_unlock(&cache->dirty_tail_lock); + } else { + /* If cache dont need writeback, then there is no + * dirty key, that means the dirty_tail is key_head + */ + spin_lock(&cache->key_head_lock); + cache_pos_copy(&dirty_tail, &cache->key_head); + spin_unlock(&cache->key_head_lock); + } mutex_lock(&cache->key_tail_lock); cache_pos_copy(&key_tail, &cache->key_tail); @@ -152,6 +161,9 @@ void pcache_cache_gc_fn(struct work_struct *work) return; } + if (cache_key_clear(key)) + continue; + cache_key_gc(cache, key); } diff --git a/drivers/md/dm-pcache/cache_key.c b/drivers/md/dm-pcache/cache_key.c index 2b77e121f89be9..33681c1cbbe287 100644 --- a/drivers/md/dm-pcache/cache_key.c +++ b/drivers/md/dm-pcache/cache_key.c @@ -74,12 +74,14 @@ static void cache_key_encode(struct pcache_cache *cache, { key_onmedia->off = key->off; key_onmedia->len = key->len; + key_onmedia->flags = key->flags; + + if (cache_key_clear(key)) + return; key_onmedia->cache_seg_id = key->cache_pos.cache_seg->cache_seg_id; key_onmedia->cache_seg_off = key->cache_pos.seg_off; - key_onmedia->seg_gen = key->seg_gen; - key_onmedia->flags = key->flags; if (cache_data_crc_on(cache)) key_onmedia->data_crc = cache_key_data_crc(key); @@ -93,12 +95,14 @@ int cache_key_decode(struct pcache_cache *cache, key->off = key_onmedia->off; key->len = key_onmedia->len; + key->flags = key_onmedia->flags; + + if (cache_key_clear(key)) + return 0; key->cache_pos.cache_seg = &cache->segments[key_onmedia->cache_seg_id]; key->cache_pos.seg_off = key_onmedia->cache_seg_off; - key->seg_gen = key_onmedia->seg_gen; - key->flags = key_onmedia->flags; if (cache_data_crc_on(cache) && key_onmedia->data_crc != cache_key_data_crc(key)) { @@ -628,6 +632,9 @@ void cache_key_insert(struct pcache_cache_tree *cache_tree, struct pcache_cache_ if (walk_ctx.pre_alloc_key) cache_key_put(walk_ctx.pre_alloc_key); + if (cache_key_clear(key)) + return; + /* Link and insert the new key into the red-black tree */ rb_link_node(&key->rb_node, parent, new); rb_insert_color(&key->rb_node, &cache_subtree->root); @@ -728,6 +735,15 @@ static int kset_replay(struct pcache_cache *cache, struct pcache_cache_kset_onme goto err; } + if (cache_key_clear(key)) { + cache_subtree = get_subtree(&cache->req_key_tree, key->off); + spin_lock(&cache_subtree->tree_lock); + cache_key_insert(&cache->req_key_tree, key, true); + cache_key_put(key); + spin_unlock(&cache_subtree->tree_lock); + continue; + } + __set_bit(key->cache_pos.cache_seg->cache_seg_id, cache->seg_map); /* Check if the segment generation is valid for insertion. */ diff --git a/drivers/md/dm-pcache/cache_req.c b/drivers/md/dm-pcache/cache_req.c index 7854a30e07b7f5..8684b048a659a6 100644 --- a/drivers/md/dm-pcache/cache_req.c +++ b/drivers/md/dm-pcache/cache_req.c @@ -272,7 +272,7 @@ static void cache_miss_req_init(struct pcache_cache *cache, backing_dev_req_init(backing_req, &req_opts); - if (insert_key) { + if (insert_key && cache_mode_need_read(cache)) { key = backing_req->priv_data; key->off = parent->off + off; key->len = len; @@ -736,7 +736,39 @@ static int cache_read(struct pcache_cache *cache, struct pcache_request *pcache_ return ret; } -static int cache_write(struct pcache_cache *cache, struct pcache_request *pcache_req) +static int cache_data_write(struct pcache_cache *cache, struct pcache_request *pcache_req); +static void backing_write_end_req(struct pcache_backing_dev_req *backing_req, int ret) +{ + struct pcache_request *pcache_req = backing_req->req.upper_req; + struct pcache_cache *cache = backing_req->backing_dev->cache; + + if (ret) + return; + + if (!cache_mode_need_cache(cache)) { + ret = cache_data_write(cache, pcache_req); + if (ret && !pcache_req->ret) + pcache_req->ret = ret; + } +} + +static void backing_write_req_send(struct pcache_backing_dev *backing_dev, + struct pcache_request *pcache_req) +{ + struct pcache_backing_dev_req *backing_req; + struct pcache_backing_dev_req_opts req_opts = { 0 }; + + req_opts.type = BACKING_DEV_REQ_TYPE_REQ; + req_opts.req.upper_req = pcache_req; + req_opts.req.req_off = 0; + req_opts.req.len = pcache_req->data_len; + req_opts.end_fn = backing_write_end_req; + + backing_req = backing_dev_req_create(backing_dev, &req_opts); + backing_dev_req_submit(backing_req, true); +} + +static int cache_data_write(struct pcache_cache *cache, struct pcache_request *pcache_req) { struct pcache_cache_subtree *cache_subtree; struct pcache_cache_key *key; @@ -755,17 +787,25 @@ static int cache_write(struct pcache_cache *cache, struct pcache_request *pcache if (key->len > PCACHE_CACHE_SUBTREE_SIZE - (key->off & PCACHE_CACHE_SUBTREE_SIZE_MASK)) key->len = PCACHE_CACHE_SUBTREE_SIZE - (key->off & PCACHE_CACHE_SUBTREE_SIZE_MASK); - ret = cache_data_alloc(cache, key); - if (ret) { - cache_key_put(key); - goto err; - } + if (cache_mode_need_cache(cache)) { + ret = cache_data_alloc(cache, key); + if (ret) { + cache_key_put(key); + goto err; + } - ret = cache_copy_from_req_bio(cache, key, pcache_req, io_done); - if (ret) { - cache_seg_put(key->cache_pos.cache_seg); - cache_key_put(key); - goto err; + if (!cache_mode_need_writeback(cache)) + key->flags |= PCACHE_CACHE_KEY_FLAGS_CLEAN; + + ret = cache_copy_from_req_bio(cache, key, pcache_req, io_done); + if (ret) { + cache_seg_put(key->cache_pos.cache_seg); + cache_key_put(key); + goto err; + } + } else { + /* clear the range to write */ + key->flags |= PCACHE_CACHE_KEY_FLAGS_CLEAR; } cache_subtree = get_subtree(&cache->req_key_tree, key->off); @@ -779,9 +819,10 @@ static int cache_write(struct pcache_cache *cache, struct pcache_request *pcache } io_done += key->len; + if (cache_key_clear(key)) + cache_key_put(key); spin_unlock(&cache_subtree->tree_lock); } - return 0; unlock: spin_unlock(&cache_subtree->tree_lock); @@ -789,6 +830,19 @@ static int cache_write(struct pcache_cache *cache, struct pcache_request *pcache return ret; } +static int cache_write(struct pcache_cache *cache, struct pcache_request *pcache_req) +{ + int ret = 0; + + if (cache_mode_need_backing(cache)) + backing_write_req_send(cache->backing_dev, pcache_req); + + if (cache_mode_need_cache(cache)) + ret = cache_data_write(cache, pcache_req); + + return ret; +} + /** * pcache_cache_flush - Flush all ksets to persist any pending cache data * @cache: Pointer to the cache structure diff --git a/drivers/md/dm-pcache/cache_writeback.c b/drivers/md/dm-pcache/cache_writeback.c index 87a82b3fe8363a..b27a16eaf6af18 100644 --- a/drivers/md/dm-pcache/cache_writeback.c +++ b/drivers/md/dm-pcache/cache_writeback.c @@ -76,6 +76,9 @@ static inline bool is_cache_clean(struct pcache_cache *cache, struct pcache_cach void cache_writeback_exit(struct pcache_cache *cache) { + if (!cache_mode_need_writeback(cache)) + return; + cancel_delayed_work_sync(&cache->writeback_work); backing_dev_flush(cache->backing_dev); cache_tree_exit(&cache->writeback_key_tree); @@ -85,6 +88,9 @@ int cache_writeback_init(struct pcache_cache *cache) { int ret; + if (!cache_mode_need_writeback(cache)) + return 0; + ret = cache_tree_init(cache, &cache->writeback_key_tree, 1); if (ret) goto err; diff --git a/drivers/md/dm-pcache/dm_pcache.c b/drivers/md/dm-pcache/dm_pcache.c index e5f5936fa6f08f..b709b4fd376d6a 100644 --- a/drivers/md/dm-pcache/dm_pcache.c +++ b/drivers/md/dm-pcache/dm_pcache.c @@ -171,6 +171,12 @@ static int parse_cache_opts(struct dm_pcache *pcache, struct dm_arg_set *as, arg = dm_shift_arg(as); if (!strcmp(arg, "writeback")) { opts->cache_mode = PCACHE_CACHE_MODE_WRITEBACK; + } else if (!strcmp(arg, "writethrough")) { + opts->cache_mode = PCACHE_CACHE_MODE_WRITETHROUGH; + } else if (!strcmp(arg, "writearound")) { + opts->cache_mode = PCACHE_CACHE_MODE_WRITEAROUND; + } else if (!strcmp(arg, "writeonly")) { + opts->cache_mode = PCACHE_CACHE_MODE_WRITEONLY; } else { *error = "Invalid cache mode parameter"; return -EINVAL; @@ -416,9 +422,10 @@ static void dm_pcache_status(struct dm_target *ti, status_type_t type, cache->key_tail.seg_off); break; case STATUSTYPE_TABLE: - DMEMIT("%s %s 4 cache_mode writeback crc %s", + DMEMIT("%s %s 4 cache_mode %s crc %s", cache_dev->dm_dev->name, backing_dev->dm_dev->name, + cache_mode_str(cache_mode_get(cache)), cache_data_crc_on(cache) ? "true" : "false"); break; case STATUSTYPE_IMA: