316 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * f2fs iostat support4 *5 * Copyright 2021 Google LLC6 * Author: Daeho Jeong <daehojeong@google.com>7 */8 9#include <linux/fs.h>10#include <linux/f2fs_fs.h>11#include <linux/seq_file.h>12 13#include "f2fs.h"14#include "iostat.h"15#include <trace/events/f2fs.h>16 17static struct kmem_cache *bio_iostat_ctx_cache;18static mempool_t *bio_iostat_ctx_pool;19 20static inline unsigned long long iostat_get_avg_bytes(struct f2fs_sb_info *sbi,21 enum iostat_type type)22{23 return sbi->iostat_count[type] ? div64_u64(sbi->iostat_bytes[type],24 sbi->iostat_count[type]) : 0;25}26 27#define IOSTAT_INFO_SHOW(name, type) \28 seq_printf(seq, "%-23s %-16llu %-16llu %-16llu\n", \29 name":", sbi->iostat_bytes[type], \30 sbi->iostat_count[type], \31 iostat_get_avg_bytes(sbi, type))32 33int __maybe_unused iostat_info_seq_show(struct seq_file *seq, void *offset)34{35 struct super_block *sb = seq->private;36 struct f2fs_sb_info *sbi = F2FS_SB(sb);37 38 if (!sbi->iostat_enable)39 return 0;40 41 seq_printf(seq, "time: %-16llu\n", ktime_get_real_seconds());42 seq_printf(seq, "\t\t\t%-16s %-16s %-16s\n",43 "io_bytes", "count", "avg_bytes");44 45 /* print app write IOs */46 seq_puts(seq, "[WRITE]\n");47 IOSTAT_INFO_SHOW("app buffered data", APP_BUFFERED_IO);48 IOSTAT_INFO_SHOW("app direct data", APP_DIRECT_IO);49 IOSTAT_INFO_SHOW("app mapped data", APP_MAPPED_IO);50 IOSTAT_INFO_SHOW("app buffered cdata", APP_BUFFERED_CDATA_IO);51 IOSTAT_INFO_SHOW("app mapped cdata", APP_MAPPED_CDATA_IO);52 53 /* print fs write IOs */54 IOSTAT_INFO_SHOW("fs data", FS_DATA_IO);55 IOSTAT_INFO_SHOW("fs cdata", FS_CDATA_IO);56 IOSTAT_INFO_SHOW("fs node", FS_NODE_IO);57 IOSTAT_INFO_SHOW("fs meta", FS_META_IO);58 IOSTAT_INFO_SHOW("fs gc data", FS_GC_DATA_IO);59 IOSTAT_INFO_SHOW("fs gc node", FS_GC_NODE_IO);60 IOSTAT_INFO_SHOW("fs cp data", FS_CP_DATA_IO);61 IOSTAT_INFO_SHOW("fs cp node", FS_CP_NODE_IO);62 IOSTAT_INFO_SHOW("fs cp meta", FS_CP_META_IO);63 64 /* print app read IOs */65 seq_puts(seq, "[READ]\n");66 IOSTAT_INFO_SHOW("app buffered data", APP_BUFFERED_READ_IO);67 IOSTAT_INFO_SHOW("app direct data", APP_DIRECT_READ_IO);68 IOSTAT_INFO_SHOW("app mapped data", APP_MAPPED_READ_IO);69 IOSTAT_INFO_SHOW("app buffered cdata", APP_BUFFERED_CDATA_READ_IO);70 IOSTAT_INFO_SHOW("app mapped cdata", APP_MAPPED_CDATA_READ_IO);71 72 /* print fs read IOs */73 IOSTAT_INFO_SHOW("fs data", FS_DATA_READ_IO);74 IOSTAT_INFO_SHOW("fs gc data", FS_GDATA_READ_IO);75 IOSTAT_INFO_SHOW("fs cdata", FS_CDATA_READ_IO);76 IOSTAT_INFO_SHOW("fs node", FS_NODE_READ_IO);77 IOSTAT_INFO_SHOW("fs meta", FS_META_READ_IO);78 79 /* print other IOs */80 seq_puts(seq, "[OTHER]\n");81 IOSTAT_INFO_SHOW("fs discard", FS_DISCARD_IO);82 IOSTAT_INFO_SHOW("fs flush", FS_FLUSH_IO);83 IOSTAT_INFO_SHOW("fs zone reset", FS_ZONE_RESET_IO);84 85 return 0;86}87 88static inline void __record_iostat_latency(struct f2fs_sb_info *sbi)89{90 int io, idx;91 struct f2fs_iostat_latency iostat_lat[MAX_IO_TYPE][NR_PAGE_TYPE];92 struct iostat_lat_info *io_lat = sbi->iostat_io_lat;93 unsigned long flags;94 95 spin_lock_irqsave(&sbi->iostat_lat_lock, flags);96 for (idx = 0; idx < MAX_IO_TYPE; idx++) {97 for (io = 0; io < NR_PAGE_TYPE; io++) {98 iostat_lat[idx][io].peak_lat =99 jiffies_to_msecs(io_lat->peak_lat[idx][io]);100 iostat_lat[idx][io].cnt = io_lat->bio_cnt[idx][io];101 iostat_lat[idx][io].avg_lat = iostat_lat[idx][io].cnt ?102 jiffies_to_msecs(io_lat->sum_lat[idx][io]) / iostat_lat[idx][io].cnt : 0;103 io_lat->sum_lat[idx][io] = 0;104 io_lat->peak_lat[idx][io] = 0;105 io_lat->bio_cnt[idx][io] = 0;106 }107 }108 spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags);109 110 trace_f2fs_iostat_latency(sbi, iostat_lat);111}112 113static inline void f2fs_record_iostat(struct f2fs_sb_info *sbi)114{115 unsigned long long iostat_diff[NR_IO_TYPE];116 int i;117 unsigned long flags;118 119 if (time_is_after_jiffies(sbi->iostat_next_period))120 return;121 122 /* Need double check under the lock */123 spin_lock_irqsave(&sbi->iostat_lock, flags);124 if (time_is_after_jiffies(sbi->iostat_next_period)) {125 spin_unlock_irqrestore(&sbi->iostat_lock, flags);126 return;127 }128 sbi->iostat_next_period = jiffies +129 msecs_to_jiffies(sbi->iostat_period_ms);130 131 for (i = 0; i < NR_IO_TYPE; i++) {132 iostat_diff[i] = sbi->iostat_bytes[i] -133 sbi->prev_iostat_bytes[i];134 sbi->prev_iostat_bytes[i] = sbi->iostat_bytes[i];135 }136 spin_unlock_irqrestore(&sbi->iostat_lock, flags);137 138 trace_f2fs_iostat(sbi, iostat_diff);139 140 __record_iostat_latency(sbi);141}142 143void f2fs_reset_iostat(struct f2fs_sb_info *sbi)144{145 struct iostat_lat_info *io_lat = sbi->iostat_io_lat;146 int i;147 148 spin_lock_irq(&sbi->iostat_lock);149 for (i = 0; i < NR_IO_TYPE; i++) {150 sbi->iostat_count[i] = 0;151 sbi->iostat_bytes[i] = 0;152 sbi->prev_iostat_bytes[i] = 0;153 }154 spin_unlock_irq(&sbi->iostat_lock);155 156 spin_lock_irq(&sbi->iostat_lat_lock);157 memset(io_lat, 0, sizeof(struct iostat_lat_info));158 spin_unlock_irq(&sbi->iostat_lat_lock);159}160 161static inline void __f2fs_update_iostat(struct f2fs_sb_info *sbi,162 enum iostat_type type, unsigned long long io_bytes)163{164 sbi->iostat_bytes[type] += io_bytes;165 sbi->iostat_count[type]++;166}167 168void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode,169 enum iostat_type type, unsigned long long io_bytes)170{171 unsigned long flags;172 173 if (!sbi->iostat_enable)174 return;175 176 spin_lock_irqsave(&sbi->iostat_lock, flags);177 __f2fs_update_iostat(sbi, type, io_bytes);178 179 if (type == APP_BUFFERED_IO || type == APP_DIRECT_IO)180 __f2fs_update_iostat(sbi, APP_WRITE_IO, io_bytes);181 182 if (type == APP_BUFFERED_READ_IO || type == APP_DIRECT_READ_IO)183 __f2fs_update_iostat(sbi, APP_READ_IO, io_bytes);184 185#ifdef CONFIG_F2FS_FS_COMPRESSION186 if (inode && f2fs_compressed_file(inode)) {187 if (type == APP_BUFFERED_IO)188 __f2fs_update_iostat(sbi, APP_BUFFERED_CDATA_IO, io_bytes);189 190 if (type == APP_BUFFERED_READ_IO)191 __f2fs_update_iostat(sbi, APP_BUFFERED_CDATA_READ_IO, io_bytes);192 193 if (type == APP_MAPPED_READ_IO)194 __f2fs_update_iostat(sbi, APP_MAPPED_CDATA_READ_IO, io_bytes);195 196 if (type == APP_MAPPED_IO)197 __f2fs_update_iostat(sbi, APP_MAPPED_CDATA_IO, io_bytes);198 199 if (type == FS_DATA_READ_IO)200 __f2fs_update_iostat(sbi, FS_CDATA_READ_IO, io_bytes);201 202 if (type == FS_DATA_IO)203 __f2fs_update_iostat(sbi, FS_CDATA_IO, io_bytes);204 }205#endif206 207 spin_unlock_irqrestore(&sbi->iostat_lock, flags);208 209 f2fs_record_iostat(sbi);210}211 212static inline void __update_iostat_latency(struct bio_iostat_ctx *iostat_ctx,213 enum iostat_lat_type lat_type)214{215 unsigned long ts_diff;216 unsigned int page_type = iostat_ctx->type;217 struct f2fs_sb_info *sbi = iostat_ctx->sbi;218 struct iostat_lat_info *io_lat = sbi->iostat_io_lat;219 unsigned long flags;220 221 if (!sbi->iostat_enable)222 return;223 224 ts_diff = jiffies - iostat_ctx->submit_ts;225 if (page_type == META_FLUSH) {226 page_type = META;227 } else if (page_type >= NR_PAGE_TYPE) {228 f2fs_warn(sbi, "%s: %d over NR_PAGE_TYPE", __func__, page_type);229 return;230 }231 232 spin_lock_irqsave(&sbi->iostat_lat_lock, flags);233 io_lat->sum_lat[lat_type][page_type] += ts_diff;234 io_lat->bio_cnt[lat_type][page_type]++;235 if (ts_diff > io_lat->peak_lat[lat_type][page_type])236 io_lat->peak_lat[lat_type][page_type] = ts_diff;237 spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags);238}239 240void iostat_update_and_unbind_ctx(struct bio *bio)241{242 struct bio_iostat_ctx *iostat_ctx = bio->bi_private;243 enum iostat_lat_type lat_type;244 245 if (op_is_write(bio_op(bio))) {246 lat_type = bio->bi_opf & REQ_SYNC ?247 WRITE_SYNC_IO : WRITE_ASYNC_IO;248 bio->bi_private = iostat_ctx->sbi;249 } else {250 lat_type = READ_IO;251 bio->bi_private = iostat_ctx->post_read_ctx;252 }253 254 __update_iostat_latency(iostat_ctx, lat_type);255 mempool_free(iostat_ctx, bio_iostat_ctx_pool);256}257 258void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi,259 struct bio *bio, struct bio_post_read_ctx *ctx)260{261 struct bio_iostat_ctx *iostat_ctx;262 /* Due to the mempool, this never fails. */263 iostat_ctx = mempool_alloc(bio_iostat_ctx_pool, GFP_NOFS);264 iostat_ctx->sbi = sbi;265 iostat_ctx->submit_ts = 0;266 iostat_ctx->type = 0;267 iostat_ctx->post_read_ctx = ctx;268 bio->bi_private = iostat_ctx;269}270 271int __init f2fs_init_iostat_processing(void)272{273 bio_iostat_ctx_cache =274 kmem_cache_create("f2fs_bio_iostat_ctx",275 sizeof(struct bio_iostat_ctx), 0, 0, NULL);276 if (!bio_iostat_ctx_cache)277 goto fail;278 bio_iostat_ctx_pool =279 mempool_create_slab_pool(NUM_PREALLOC_IOSTAT_CTXS,280 bio_iostat_ctx_cache);281 if (!bio_iostat_ctx_pool)282 goto fail_free_cache;283 return 0;284 285fail_free_cache:286 kmem_cache_destroy(bio_iostat_ctx_cache);287fail:288 return -ENOMEM;289}290 291void f2fs_destroy_iostat_processing(void)292{293 mempool_destroy(bio_iostat_ctx_pool);294 kmem_cache_destroy(bio_iostat_ctx_cache);295}296 297int f2fs_init_iostat(struct f2fs_sb_info *sbi)298{299 /* init iostat info */300 spin_lock_init(&sbi->iostat_lock);301 spin_lock_init(&sbi->iostat_lat_lock);302 sbi->iostat_enable = false;303 sbi->iostat_period_ms = DEFAULT_IOSTAT_PERIOD_MS;304 sbi->iostat_io_lat = f2fs_kzalloc(sbi, sizeof(struct iostat_lat_info),305 GFP_KERNEL);306 if (!sbi->iostat_io_lat)307 return -ENOMEM;308 309 return 0;310}311 312void f2fs_destroy_iostat(struct f2fs_sb_info *sbi)313{314 kfree(sbi->iostat_io_lat);315}316