150 lines · c
1/* SPDX-License-Identifier: GPL-2.02 *3 * Legacy blkg rwstat helpers enabled by CONFIG_BLK_CGROUP_RWSTAT.4 * Do not use in new code.5 */6#ifndef _BLK_CGROUP_RWSTAT_H7#define _BLK_CGROUP_RWSTAT_H8 9#include "blk-cgroup.h"10 11enum blkg_rwstat_type {12 BLKG_RWSTAT_READ,13 BLKG_RWSTAT_WRITE,14 BLKG_RWSTAT_SYNC,15 BLKG_RWSTAT_ASYNC,16 BLKG_RWSTAT_DISCARD,17 18 BLKG_RWSTAT_NR,19 BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,20};21 22/*23 * blkg_[rw]stat->aux_cnt is excluded for local stats but included for24 * recursive. Used to carry stats of dead children.25 */26struct blkg_rwstat {27 struct percpu_counter cpu_cnt[BLKG_RWSTAT_NR];28 atomic64_t aux_cnt[BLKG_RWSTAT_NR];29};30 31struct blkg_rwstat_sample {32 u64 cnt[BLKG_RWSTAT_NR];33};34 35static inline u64 blkg_rwstat_read_counter(struct blkg_rwstat *rwstat,36 unsigned int idx)37{38 return atomic64_read(&rwstat->aux_cnt[idx]) +39 percpu_counter_sum_positive(&rwstat->cpu_cnt[idx]);40}41 42int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp);43void blkg_rwstat_exit(struct blkg_rwstat *rwstat);44u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,45 const struct blkg_rwstat_sample *rwstat);46u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,47 int off);48void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol,49 int off, struct blkg_rwstat_sample *sum);50 51 52/**53 * blkg_rwstat_add - add a value to a blkg_rwstat54 * @rwstat: target blkg_rwstat55 * @op: REQ_OP and flags56 * @val: value to add57 *58 * Add @val to @rwstat. The counters are chosen according to @rw. The59 * caller is responsible for synchronizing calls to this function.60 */61static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,62 blk_opf_t opf, uint64_t val)63{64 struct percpu_counter *cnt;65 66 if (op_is_discard(opf))67 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_DISCARD];68 else if (op_is_write(opf))69 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE];70 else71 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ];72 73 percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);74 75 if (op_is_sync(opf))76 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC];77 else78 cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC];79 80 percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);81}82 83/**84 * blkg_rwstat_read - read the current values of a blkg_rwstat85 * @rwstat: blkg_rwstat to read86 *87 * Read the current snapshot of @rwstat and return it in the aux counts.88 */89static inline void blkg_rwstat_read(struct blkg_rwstat *rwstat,90 struct blkg_rwstat_sample *result)91{92 int i;93 94 for (i = 0; i < BLKG_RWSTAT_NR; i++)95 result->cnt[i] =96 percpu_counter_sum_positive(&rwstat->cpu_cnt[i]);97}98 99/**100 * blkg_rwstat_total - read the total count of a blkg_rwstat101 * @rwstat: blkg_rwstat to read102 *103 * Return the total count of @rwstat regardless of the IO direction. This104 * function can be called without synchronization and takes care of u64105 * atomicity.106 */107static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)108{109 struct blkg_rwstat_sample tmp = { };110 111 blkg_rwstat_read(rwstat, &tmp);112 return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];113}114 115/**116 * blkg_rwstat_reset - reset a blkg_rwstat117 * @rwstat: blkg_rwstat to reset118 */119static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)120{121 int i;122 123 for (i = 0; i < BLKG_RWSTAT_NR; i++) {124 percpu_counter_set(&rwstat->cpu_cnt[i], 0);125 atomic64_set(&rwstat->aux_cnt[i], 0);126 }127}128 129/**130 * blkg_rwstat_add_aux - add a blkg_rwstat into another's aux count131 * @to: the destination blkg_rwstat132 * @from: the source133 *134 * Add @from's count including the aux one to @to's aux count.135 */136static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to,137 struct blkg_rwstat *from)138{139 u64 sum[BLKG_RWSTAT_NR];140 int i;141 142 for (i = 0; i < BLKG_RWSTAT_NR; i++)143 sum[i] = percpu_counter_sum_positive(&from->cpu_cnt[i]);144 145 for (i = 0; i < BLKG_RWSTAT_NR; i++)146 atomic64_add(sum[i] + atomic64_read(&from->aux_cnt[i]),147 &to->aux_cnt[i]);148}149#endif /* _BLK_CGROUP_RWSTAT_H */150