zfs/cmd/arc_summary/arc_summary2

#!/usr/bin/env python2
#
# $Id: arc_summary.pl,v 388:e27800740aa2 2011-07-08 02:53:29Z jhell $
#
# Copyright (c) 2008 Ben Rockwood <benr@cuddletech.com>,
# Copyright (c) 2010 Martin Matuska <mm@FreeBSD.org>,
# Copyright (c) 2010-2011 Jason J. Hellenthal <jhell@DataIX.net>,
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
# If you are having troubles when using this script from cron(8) please try
# adjusting your PATH before reporting problems.
#
# Note some of this code uses older code (eg getopt instead of argparse,
# subprocess.Popen() instead of subprocess.run()) because we need to support
# some very old versions of Python.
#

"""Print statistics on the ZFS Adjustable Replacement Cache (ARC)

Provides basic information on the ARC, its efficiency, the L2ARC (if present),
the Data Management Unit (DMU), Virtual Devices (VDEVs), and tunables. See the
in-source documentation and code at
https://github.com/openzfs/zfs/blob/master/module/zfs/arc.c for details.
"""

import getopt
import os
import sys
import time
import errno

from subprocess import Popen, PIPE
from decimal import Decimal as D


if sys.platform.startswith('freebsd'):
    # Requires py27-sysctl on FreeBSD
    import sysctl

    def is_value(ctl):
        return ctl.type != sysctl.CTLTYPE_NODE

    def load_kstats(namespace):
        """Collect information on a specific subsystem of the ARC"""

        base = 'kstat.zfs.misc.%s.' % namespace
        fmt = lambda kstat: (kstat.name, D(kstat.value))
        kstats = sysctl.filter(base)
        return [fmt(kstat) for kstat in kstats if is_value(kstat)]

    def load_tunables():
        ctls = sysctl.filter('vfs.zfs')
        return dict((ctl.name, ctl.value) for ctl in ctls if is_value(ctl))

elif sys.platform.startswith('linux'):

    def load_kstats(namespace):
        """Collect information on a specific subsystem of the ARC"""

        kstat = 'kstat.zfs.misc.%s.%%s' % namespace
        path = '/proc/spl/kstat/zfs/%s' % namespace
        with open(path) as f:
            entries = [line.strip().split() for line in f][2:] # Skip header
        return [(kstat % name, D(value)) for name, _, value in entries]

    def load_tunables():
        basepath = '/sys/module/zfs/parameters'
        tunables = {}
        for name in os.listdir(basepath):
            if not name:
                continue
            path = '%s/%s' % (basepath, name)
            with open(path) as f:
                value = f.read()
            tunables[name] = value.strip()
        return tunables


show_tunable_descriptions = False
alternate_tunable_layout = False


def handle_Exception(ex_cls, ex, tb):
    if ex is IOError:
        if ex.errno == errno.EPIPE:
            sys.exit()

    if ex is KeyboardInterrupt:
        sys.exit()


sys.excepthook = handle_Exception


def get_Kstat():
    """Collect information on the ZFS subsystem from the /proc virtual
    file system. The name "kstat" is a holdover from the Solaris utility
    of the same name.
    """

    Kstat = {}
    Kstat.update(load_kstats('arcstats'))
    Kstat.update(load_kstats('zfetchstats'))
    Kstat.update(load_kstats('vdev_cache_stats'))
    return Kstat


def fBytes(b=0):
    """Return human-readable representation of a byte value in
    powers of 2 (eg "KiB" for "kibibytes", etc) to two decimal
    points. Values smaller than one KiB are returned without
    decimal points.
    """

    prefixes = [
        [2**80, "YiB"],   # yobibytes (yotta)
        [2**70, "ZiB"],   # zebibytes (zetta)
        [2**60, "EiB"],   # exbibytes (exa)
        [2**50, "PiB"],   # pebibytes (peta)
        [2**40, "TiB"],   # tebibytes (tera)
        [2**30, "GiB"],   # gibibytes (giga)
        [2**20, "MiB"],   # mebibytes (mega)
        [2**10, "KiB"]]   # kibibytes (kilo)

    if b >= 2**10:

        for limit, unit in prefixes:

            if b >= limit:
                value = b / limit
                break

        result = "%0.2f\t%s" % (value, unit)

    else:

        result = "%d\tBytes" % b

    return result


def fHits(hits=0):
    """Create a human-readable representation of the number of hits.
    The single-letter symbols used are SI to avoid the confusion caused
    by the different "short scale" and "long scale" representations in
    English, which use the same words for different values. See
    https://en.wikipedia.org/wiki/Names_of_large_numbers and
    https://physics.nist.gov/cuu/Units/prefixes.html
    """

    numbers = [
            [10**24, 'Y'],  # yotta (septillion)
            [10**21, 'Z'],  # zetta (sextillion)
            [10**18, 'E'],  # exa   (quintrillion)
            [10**15, 'P'],  # peta  (quadrillion)
            [10**12, 'T'],  # tera  (trillion)
            [10**9, 'G'],   # giga  (billion)
            [10**6, 'M'],   # mega  (million)
            [10**3, 'k']]   # kilo  (thousand)

    if hits >= 1000:

        for limit, symbol in numbers:

            if hits >= limit:
                value = hits/limit
                break

        result = "%0.2f%s" % (value, symbol)

    else:

        result = "%d" % hits

    return result


def fPerc(lVal=0, rVal=0, Decimal=2):
    """Calculate percentage value and return in human-readable format"""

    if rVal > 0:
        return str("%0." + str(Decimal) + "f") % (100 * (lVal / rVal)) + "%"
    else:
        return str("%0." + str(Decimal) + "f") % 100 + "%"


def get_arc_summary(Kstat):
    """Collect general data on the ARC"""

    output = {}
    memory_throttle_count = Kstat[
        "kstat.zfs.misc.arcstats.memory_throttle_count"
        ]

    if memory_throttle_count > 0:
        output['health'] = 'THROTTLED'
    else:
        output['health'] = 'HEALTHY'

    output['memory_throttle_count'] = fHits(memory_throttle_count)

    # ARC Misc.
    deleted = Kstat["kstat.zfs.misc.arcstats.deleted"]
    mutex_miss = Kstat["kstat.zfs.misc.arcstats.mutex_miss"]
    evict_skip = Kstat["kstat.zfs.misc.arcstats.evict_skip"]

    # ARC Misc.
    output["arc_misc"] = {}
    output["arc_misc"]["deleted"] = fHits(deleted)
    output["arc_misc"]['mutex_miss'] = fHits(mutex_miss)
    output["arc_misc"]['evict_skips'] = fHits(evict_skip)

    # ARC Sizing
    arc_size = Kstat["kstat.zfs.misc.arcstats.size"]
    mru_size = Kstat["kstat.zfs.misc.arcstats.mru_size"]
    mfu_size = Kstat["kstat.zfs.misc.arcstats.mfu_size"]
    meta_limit = Kstat["kstat.zfs.misc.arcstats.arc_meta_limit"]
    meta_size = Kstat["kstat.zfs.misc.arcstats.arc_meta_used"]
    dnode_limit = Kstat["kstat.zfs.misc.arcstats.arc_dnode_limit"]
    dnode_size = Kstat["kstat.zfs.misc.arcstats.dnode_size"]
    target_max_size = Kstat["kstat.zfs.misc.arcstats.c_max"]
    target_min_size = Kstat["kstat.zfs.misc.arcstats.c_min"]
    target_size = Kstat["kstat.zfs.misc.arcstats.c"]

    target_size_ratio = (target_max_size / target_min_size)

    # ARC Sizing
    output['arc_sizing'] = {}
    output['arc_sizing']['arc_size'] = {
        'per': fPerc(arc_size, target_max_size),
        'num': fBytes(arc_size),
    }
    output['arc_sizing']['target_max_size'] = {
        'ratio': target_size_ratio,
        'num': fBytes(target_max_size),
    }
    output['arc_sizing']['target_min_size'] = {
        'per': fPerc(target_min_size, target_max_size),
        'num': fBytes(target_min_size),
    }
    output['arc_sizing']['target_size'] = {
        'per': fPerc(target_size, target_max_size),
        'num': fBytes(target_size),
    }
    output['arc_sizing']['meta_limit'] = {
        'per': fPerc(meta_limit, target_max_size),
        'num': fBytes(meta_limit),
    }
    output['arc_sizing']['meta_size'] = {
        'per': fPerc(meta_size, meta_limit),
        'num': fBytes(meta_size),
    }
    output['arc_sizing']['dnode_limit'] = {
        'per': fPerc(dnode_limit, meta_limit),
        'num': fBytes(dnode_limit),
    }
    output['arc_sizing']['dnode_size'] = {
        'per': fPerc(dnode_size, dnode_limit),
        'num': fBytes(dnode_size),
    }

    # ARC Hash Breakdown
    output['arc_hash_break'] = {}
    output['arc_hash_break']['hash_chain_max'] = Kstat[
        "kstat.zfs.misc.arcstats.hash_chain_max"
        ]
    output['arc_hash_break']['hash_chains'] = Kstat[
        "kstat.zfs.misc.arcstats.hash_chains"
        ]
    output['arc_hash_break']['hash_collisions'] = Kstat[
        "kstat.zfs.misc.arcstats.hash_collisions"
        ]
    output['arc_hash_break']['hash_elements'] = Kstat[
        "kstat.zfs.misc.arcstats.hash_elements"
        ]
    output['arc_hash_break']['hash_elements_max'] = Kstat[
        "kstat.zfs.misc.arcstats.hash_elements_max"
        ]

    output['arc_size_break'] = {}
    output['arc_size_break']['recently_used_cache_size'] = {
        'per': fPerc(mru_size, mru_size + mfu_size),
        'num': fBytes(mru_size),
    }
    output['arc_size_break']['frequently_used_cache_size'] = {
        'per': fPerc(mfu_size, mru_size + mfu_size),
        'num': fBytes(mfu_size),
    }

    # ARC Hash Breakdown
    hash_chain_max = Kstat["kstat.zfs.misc.arcstats.hash_chain_max"]
    hash_chains = Kstat["kstat.zfs.misc.arcstats.hash_chains"]
    hash_collisions = Kstat["kstat.zfs.misc.arcstats.hash_collisions"]
    hash_elements = Kstat["kstat.zfs.misc.arcstats.hash_elements"]
    hash_elements_max = Kstat["kstat.zfs.misc.arcstats.hash_elements_max"]

    output['arc_hash_break'] = {}
    output['arc_hash_break']['elements_max'] = fHits(hash_elements_max)
    output['arc_hash_break']['elements_current'] = {
        'per': fPerc(hash_elements, hash_elements_max),
        'num': fHits(hash_elements),
        }
    output['arc_hash_break']['collisions'] = fHits(hash_collisions)
    output['arc_hash_break']['chain_max'] = fHits(hash_chain_max)
    output['arc_hash_break']['chains'] = fHits(hash_chains)

    return output


def _arc_summary(Kstat):
    """Print information on the ARC"""

    # ARC Sizing
    arc = get_arc_summary(Kstat)

    sys.stdout.write("ARC Summary: (%s)\n" % arc['health'])

    sys.stdout.write("\tMemory Throttle Count:\t\t\t%s\n" %
                     arc['memory_throttle_count'])
    sys.stdout.write("\n")

    # ARC Misc.
    sys.stdout.write("ARC Misc:\n")
    sys.stdout.write("\tDeleted:\t\t\t\t%s\n" % arc['arc_misc']['deleted'])
    sys.stdout.write("\tMutex Misses:\t\t\t\t%s\n" %
                     arc['arc_misc']['mutex_miss'])
    sys.stdout.write("\tEvict Skips:\t\t\t\t%s\n" %
                     arc['arc_misc']['evict_skips'])
    sys.stdout.write("\n")

    # ARC Sizing
    sys.stdout.write("ARC Size:\t\t\t\t%s\t%s\n" % (
        arc['arc_sizing']['arc_size']['per'],
        arc['arc_sizing']['arc_size']['num']
        )
    )
    sys.stdout.write("\tTarget Size: (Adaptive)\t\t%s\t%s\n" % (
        arc['arc_sizing']['target_size']['per'],
        arc['arc_sizing']['target_size']['num'],
        )
    )

    sys.stdout.write("\tMin Size (Hard Limit):\t\t%s\t%s\n" % (
        arc['arc_sizing']['target_min_size']['per'],
        arc['arc_sizing']['target_min_size']['num'],
        )
    )

    sys.stdout.write("\tMax Size (High Water):\t\t%d:1\t%s\n" % (
        arc['arc_sizing']['target_max_size']['ratio'],
        arc['arc_sizing']['target_max_size']['num'],
        )
    )

    sys.stdout.write("\nARC Size Breakdown:\n")
    sys.stdout.write("\tRecently Used Cache Size:\t%s\t%s\n" % (
        arc['arc_size_break']['recently_used_cache_size']['per'],
        arc['arc_size_break']['recently_used_cache_size']['num'],
        )
    )
    sys.stdout.write("\tFrequently Used Cache Size:\t%s\t%s\n" % (
        arc['arc_size_break']['frequently_used_cache_size']['per'],
        arc['arc_size_break']['frequently_used_cache_size']['num'],
        )
    )
    sys.stdout.write("\tMetadata Size (Hard Limit):\t%s\t%s\n" % (
        arc['arc_sizing']['meta_limit']['per'],
        arc['arc_sizing']['meta_limit']['num'],
        )
    )
    sys.stdout.write("\tMetadata Size:\t\t\t%s\t%s\n" % (
        arc['arc_sizing']['meta_size']['per'],
        arc['arc_sizing']['meta_size']['num'],
        )
    )
    sys.stdout.write("\tDnode Size (Hard Limit):\t%s\t%s\n" % (
        arc['arc_sizing']['dnode_limit']['per'],
        arc['arc_sizing']['dnode_limit']['num'],
        )
    )
    sys.stdout.write("\tDnode Size:\t\t\t%s\t%s\n" % (
        arc['arc_sizing']['dnode_size']['per'],
        arc['arc_sizing']['dnode_size']['num'],
        )
    )

    sys.stdout.write("\n")

    # ARC Hash Breakdown
    sys.stdout.write("ARC Hash Breakdown:\n")
    sys.stdout.write("\tElements Max:\t\t\t\t%s\n" %
                     arc['arc_hash_break']['elements_max'])
    sys.stdout.write("\tElements Current:\t\t%s\t%s\n" % (
        arc['arc_hash_break']['elements_current']['per'],
        arc['arc_hash_break']['elements_current']['num'],
        )
    )
    sys.stdout.write("\tCollisions:\t\t\t\t%s\n" %
                     arc['arc_hash_break']['collisions'])
    sys.stdout.write("\tChain Max:\t\t\t\t%s\n" %
                     arc['arc_hash_break']['chain_max'])
    sys.stdout.write("\tChains:\t\t\t\t\t%s\n" %
                     arc['arc_hash_break']['chains'])


def get_arc_efficiency(Kstat):
    """Collect information on the efficiency of the ARC"""

    output = {}

    arc_hits = Kstat["kstat.zfs.misc.arcstats.hits"]
    arc_misses = Kstat["kstat.zfs.misc.arcstats.misses"]
    demand_data_hits = Kstat["kstat.zfs.misc.arcstats.demand_data_hits"]
    demand_data_misses = Kstat["kstat.zfs.misc.arcstats.demand_data_misses"]
    demand_metadata_hits = Kstat[
        "kstat.zfs.misc.arcstats.demand_metadata_hits"
        ]
    demand_metadata_misses = Kstat[
        "kstat.zfs.misc.arcstats.demand_metadata_misses"
        ]
    mfu_ghost_hits = Kstat["kstat.zfs.misc.arcstats.mfu_ghost_hits"]
    mfu_hits = Kstat["kstat.zfs.misc.arcstats.mfu_hits"]
    mru_ghost_hits = Kstat["kstat.zfs.misc.arcstats.mru_ghost_hits"]
    mru_hits = Kstat["kstat.zfs.misc.arcstats.mru_hits"]
    prefetch_data_hits = Kstat["kstat.zfs.misc.arcstats.prefetch_data_hits"]
    prefetch_data_misses = Kstat[
        "kstat.zfs.misc.arcstats.prefetch_data_misses"
        ]
    prefetch_metadata_hits = Kstat[
        "kstat.zfs.misc.arcstats.prefetch_metadata_hits"
        ]
    prefetch_metadata_misses = Kstat[
        "kstat.zfs.misc.arcstats.prefetch_metadata_misses"
        ]

    anon_hits = arc_hits - (
        mfu_hits + mru_hits + mfu_ghost_hits + mru_ghost_hits
        )
    arc_accesses_total = (arc_hits + arc_misses)
    demand_data_total = (demand_data_hits + demand_data_misses)
    prefetch_data_total = (prefetch_data_hits + prefetch_data_misses)
    real_hits = (mfu_hits + mru_hits)

    output["total_accesses"] = fHits(arc_accesses_total)
    output["cache_hit_ratio"] = {
        'per': fPerc(arc_hits, arc_accesses_total),
        'num': fHits(arc_hits),
    }
    output["cache_miss_ratio"] = {
        'per': fPerc(arc_misses, arc_accesses_total),
        'num': fHits(arc_misses),
    }
    output["actual_hit_ratio"] = {
        'per': fPerc(real_hits, arc_accesses_total),
        'num': fHits(real_hits),
    }
    output["data_demand_efficiency"] = {
        'per': fPerc(demand_data_hits, demand_data_total),
        'num': fHits(demand_data_total),
    }

    if prefetch_data_total > 0:
        output["data_prefetch_efficiency"] = {
            'per': fPerc(prefetch_data_hits, prefetch_data_total),
            'num': fHits(prefetch_data_total),
        }

    if anon_hits > 0:
        output["cache_hits_by_cache_list"] = {}
        output["cache_hits_by_cache_list"]["anonymously_used"] = {
            'per': fPerc(anon_hits, arc_hits),
            'num': fHits(anon_hits),
        }

    output["most_recently_used"] = {
        'per': fPerc(mru_hits, arc_hits),
        'num': fHits(mru_hits),
    }
    output["most_frequently_used"] = {
        'per': fPerc(mfu_hits, arc_hits),
        'num': fHits(mfu_hits),
    }
    output["most_recently_used_ghost"] = {
        'per': fPerc(mru_ghost_hits, arc_hits),
        'num': fHits(mru_ghost_hits),
    }
    output["most_frequently_used_ghost"] = {
        'per': fPerc(mfu_ghost_hits, arc_hits),
        'num': fHits(mfu_ghost_hits),
    }

    output["cache_hits_by_data_type"] = {}
    output["cache_hits_by_data_type"]["demand_data"] = {
        'per': fPerc(demand_data_hits, arc_hits),
        'num': fHits(demand_data_hits),
    }
    output["cache_hits_by_data_type"]["prefetch_data"] = {
        'per': fPerc(prefetch_data_hits, arc_hits),
        'num': fHits(prefetch_data_hits),
    }
    output["cache_hits_by_data_type"]["demand_metadata"] = {
        'per': fPerc(demand_metadata_hits, arc_hits),
        'num': fHits(demand_metadata_hits),
    }
    output["cache_hits_by_data_type"]["prefetch_metadata"] = {
        'per': fPerc(prefetch_metadata_hits, arc_hits),
        'num': fHits(prefetch_metadata_hits),
    }

    output["cache_misses_by_data_type"] = {}
    output["cache_misses_by_data_type"]["demand_data"] = {
        'per': fPerc(demand_data_misses, arc_misses),
        'num': fHits(demand_data_misses),
    }
    output["cache_misses_by_data_type"]["prefetch_data"] = {
        'per': fPerc(prefetch_data_misses, arc_misses),
        'num': fHits(prefetch_data_misses),
    }
    output["cache_misses_by_data_type"]["demand_metadata"] = {
        'per': fPerc(demand_metadata_misses, arc_misses),
        'num': fHits(demand_metadata_misses),
    }
    output["cache_misses_by_data_type"]["prefetch_metadata"] = {
        'per': fPerc(prefetch_metadata_misses, arc_misses),
        'num': fHits(prefetch_metadata_misses),
    }

    return output


def _arc_efficiency(Kstat):
    """Print information on the efficiency of the ARC"""

    arc = get_arc_efficiency(Kstat)

    sys.stdout.write("ARC Total accesses:\t\t\t\t\t%s\n" %
                     arc['total_accesses'])
    sys.stdout.write("\tCache Hit Ratio:\t\t%s\t%s\n" % (
        arc['cache_hit_ratio']['per'],
        arc['cache_hit_ratio']['num'],
        )
    )
    sys.stdout.write("\tCache Miss Ratio:\t\t%s\t%s\n" % (
        arc['cache_miss_ratio']['per'],
        arc['cache_miss_ratio']['num'],
        )
    )

    sys.stdout.write("\tActual Hit Ratio:\t\t%s\t%s\n" % (
        arc['actual_hit_ratio']['per'],
        arc['actual_hit_ratio']['num'],
        )
    )

    sys.stdout.write("\n")
    sys.stdout.write("\tData Demand Efficiency:\t\t%s\t%s\n" % (
        arc['data_demand_efficiency']['per'],
        arc['data_demand_efficiency']['num'],
        )
    )

    if 'data_prefetch_efficiency' in arc:
        sys.stdout.write("\tData Prefetch Efficiency:\t%s\t%s\n" % (
            arc['data_prefetch_efficiency']['per'],
            arc['data_prefetch_efficiency']['num'],
            )
        )
    sys.stdout.write("\n")

    sys.stdout.write("\tCACHE HITS BY CACHE LIST:\n")
    if 'cache_hits_by_cache_list' in arc:
        sys.stdout.write("\t  Anonymously Used:\t\t%s\t%s\n" % (
            arc['cache_hits_by_cache_list']['anonymously_used']['per'],
            arc['cache_hits_by_cache_list']['anonymously_used']['num'],
            )
        )
    sys.stdout.write("\t  Most Recently Used:\t\t%s\t%s\n" % (
        arc['most_recently_used']['per'],
        arc['most_recently_used']['num'],
        )
    )
    sys.stdout.write("\t  Most Frequently Used:\t\t%s\t%s\n" % (
        arc['most_frequently_used']['per'],
        arc['most_frequently_used']['num'],
        )
    )
    sys.stdout.write("\t  Most Recently Used Ghost:\t%s\t%s\n" % (
        arc['most_recently_used_ghost']['per'],
        arc['most_recently_used_ghost']['num'],
        )
    )
    sys.stdout.write("\t  Most Frequently Used Ghost:\t%s\t%s\n" % (
        arc['most_frequently_used_ghost']['per'],
        arc['most_frequently_used_ghost']['num'],
        )
    )

    sys.stdout.write("\n\tCACHE HITS BY DATA TYPE:\n")
    sys.stdout.write("\t  Demand Data:\t\t\t%s\t%s\n" % (
        arc["cache_hits_by_data_type"]['demand_data']['per'],
        arc["cache_hits_by_data_type"]['demand_data']['num'],
        )
    )
    sys.stdout.write("\t  Prefetch Data:\t\t%s\t%s\n" % (
        arc["cache_hits_by_data_type"]['prefetch_data']['per'],
        arc["cache_hits_by_data_type"]['prefetch_data']['num'],
        )
    )
    sys.stdout.write("\t  Demand Metadata:\t\t%s\t%s\n" % (
        arc["cache_hits_by_data_type"]['demand_metadata']['per'],
        arc["cache_hits_by_data_type"]['demand_metadata']['num'],
        )
    )
    sys.stdout.write("\t  Prefetch Metadata:\t\t%s\t%s\n" % (
        arc["cache_hits_by_data_type"]['prefetch_metadata']['per'],
        arc["cache_hits_by_data_type"]['prefetch_metadata']['num'],
        )
    )

    sys.stdout.write("\n\tCACHE MISSES BY DATA TYPE:\n")
    sys.stdout.write("\t  Demand Data:\t\t\t%s\t%s\n" % (
        arc["cache_misses_by_data_type"]['demand_data']['per'],
        arc["cache_misses_by_data_type"]['demand_data']['num'],
        )
    )
    sys.stdout.write("\t  Prefetch Data:\t\t%s\t%s\n" % (
        arc["cache_misses_by_data_type"]['prefetch_data']['per'],
        arc["cache_misses_by_data_type"]['prefetch_data']['num'],
        )
    )
    sys.stdout.write("\t  Demand Metadata:\t\t%s\t%s\n" % (
        arc["cache_misses_by_data_type"]['demand_metadata']['per'],
        arc["cache_misses_by_data_type"]['demand_metadata']['num'],
        )
    )
    sys.stdout.write("\t  Prefetch Metadata:\t\t%s\t%s\n" % (
        arc["cache_misses_by_data_type"]['prefetch_metadata']['per'],
        arc["cache_misses_by_data_type"]['prefetch_metadata']['num'],
        )
    )


def get_l2arc_summary(Kstat):
    """Collection information on the L2ARC"""

    output = {}

    l2_abort_lowmem = Kstat["kstat.zfs.misc.arcstats.l2_abort_lowmem"]
    l2_cksum_bad = Kstat["kstat.zfs.misc.arcstats.l2_cksum_bad"]
    l2_evict_lock_retry = Kstat["kstat.zfs.misc.arcstats.l2_evict_lock_retry"]
    l2_evict_reading = Kstat["kstat.zfs.misc.arcstats.l2_evict_reading"]
    l2_feeds = Kstat["kstat.zfs.misc.arcstats.l2_feeds"]
    l2_free_on_write = Kstat["kstat.zfs.misc.arcstats.l2_free_on_write"]
    l2_hdr_size = Kstat["kstat.zfs.misc.arcstats.l2_hdr_size"]
    l2_hits = Kstat["kstat.zfs.misc.arcstats.l2_hits"]
    l2_io_error = Kstat["kstat.zfs.misc.arcstats.l2_io_error"]
    l2_misses = Kstat["kstat.zfs.misc.arcstats.l2_misses"]
    l2_rw_clash = Kstat["kstat.zfs.misc.arcstats.l2_rw_clash"]
    l2_size = Kstat["kstat.zfs.misc.arcstats.l2_size"]
    l2_asize = Kstat["kstat.zfs.misc.arcstats.l2_asize"]
    l2_writes_done = Kstat["kstat.zfs.misc.arcstats.l2_writes_done"]
    l2_writes_error = Kstat["kstat.zfs.misc.arcstats.l2_writes_error"]
    l2_writes_sent = Kstat["kstat.zfs.misc.arcstats.l2_writes_sent"]

    l2_access_total = (l2_hits + l2_misses)
    output['l2_health_count'] = (l2_writes_error + l2_cksum_bad + l2_io_error)

    output['l2_access_total'] = l2_access_total
    output['l2_size'] = l2_size
    output['l2_asize'] = l2_asize

    if l2_size > 0 and l2_access_total > 0:

        if output['l2_health_count'] > 0:
            output["health"] = "DEGRADED"
        else:
            output["health"] = "HEALTHY"

        output["low_memory_aborts"] = fHits(l2_abort_lowmem)
        output["free_on_write"] = fHits(l2_free_on_write)
        output["rw_clashes"] = fHits(l2_rw_clash)
        output["bad_checksums"] = fHits(l2_cksum_bad)
        output["io_errors"] = fHits(l2_io_error)

        output["l2_arc_size"] = {}
        output["l2_arc_size"]["adative"] = fBytes(l2_size)
        output["l2_arc_size"]["actual"] = {
            'per': fPerc(l2_asize, l2_size),
            'num': fBytes(l2_asize)
            }
        output["l2_arc_size"]["head_size"] = {
            'per': fPerc(l2_hdr_size, l2_size),
            'num': fBytes(l2_hdr_size),
        }

        output["l2_arc_evicts"] = {}
        output["l2_arc_evicts"]['lock_retries'] = fHits(l2_evict_lock_retry)
        output["l2_arc_evicts"]['reading'] = fHits(l2_evict_reading)

        output['l2_arc_breakdown'] = {}
        output['l2_arc_breakdown']['value'] = fHits(l2_access_total)
        output['l2_arc_breakdown']['hit_ratio'] = {
            'per': fPerc(l2_hits, l2_access_total),
            'num': fHits(l2_hits),
        }
        output['l2_arc_breakdown']['miss_ratio'] = {
            'per': fPerc(l2_misses, l2_access_total),
            'num': fHits(l2_misses),
        }
        output['l2_arc_breakdown']['feeds'] = fHits(l2_feeds)

        output['l2_arc_buffer'] = {}

        output['l2_arc_writes'] = {}
        output['l2_writes_done'] = l2_writes_done
        output['l2_writes_sent'] = l2_writes_sent
        if l2_writes_done != l2_writes_sent:
            output['l2_arc_writes']['writes_sent'] = {
                'value': "FAULTED",
                'num': fHits(l2_writes_sent),
            }
            output['l2_arc_writes']['done_ratio'] = {
                'per': fPerc(l2_writes_done, l2_writes_sent),
                'num': fHits(l2_writes_done),
            }
            output['l2_arc_writes']['error_ratio'] = {
                'per': fPerc(l2_writes_error, l2_writes_sent),
                'num': fHits(l2_writes_error),
            }
        else:
            output['l2_arc_writes']['writes_sent'] = {
                'per': fPerc(100),
                'num': fHits(l2_writes_sent),
            }

    return output


def _l2arc_summary(Kstat):
    """Print information on the L2ARC"""

    arc = get_l2arc_summary(Kstat)

    if arc['l2_size'] > 0 and arc['l2_access_total'] > 0:
        sys.stdout.write("L2 ARC Summary: ")
        if arc['l2_health_count'] > 0:
            sys.stdout.write("(DEGRADED)\n")
        else:
            sys.stdout.write("(HEALTHY)\n")
        sys.stdout.write("\tLow Memory Aborts:\t\t\t%s\n" %
                         arc['low_memory_aborts'])
        sys.stdout.write("\tFree on Write:\t\t\t\t%s\n" % arc['free_on_write'])
        sys.stdout.write("\tR/W Clashes:\t\t\t\t%s\n" % arc['rw_clashes'])
        sys.stdout.write("\tBad Checksums:\t\t\t\t%s\n" % arc['bad_checksums'])
        sys.stdout.write("\tIO Errors:\t\t\t\t%s\n" % arc['io_errors'])
        sys.stdout.write("\n")

        sys.stdout.write("L2 ARC Size: (Adaptive)\t\t\t\t%s\n" %
                         arc["l2_arc_size"]["adative"])
        sys.stdout.write("\tCompressed:\t\t\t%s\t%s\n" % (
            arc["l2_arc_size"]["actual"]["per"],
            arc["l2_arc_size"]["actual"]["num"],
            )
        )
        sys.stdout.write("\tHeader Size:\t\t\t%s\t%s\n" % (
            arc["l2_arc_size"]["head_size"]["per"],
            arc["l2_arc_size"]["head_size"]["num"],
            )
        )
        sys.stdout.write("\n")

        if arc["l2_arc_evicts"]['lock_retries'] != '0' or \
           arc["l2_arc_evicts"]["reading"] != '0':
            sys.stdout.write("L2 ARC Evicts:\n")
            sys.stdout.write("\tLock Retries:\t\t\t\t%s\n" %
                             arc["l2_arc_evicts"]['lock_retries'])
            sys.stdout.write("\tUpon Reading:\t\t\t\t%s\n" %
                             arc["l2_arc_evicts"]["reading"])
            sys.stdout.write("\n")

        sys.stdout.write("L2 ARC Breakdown:\t\t\t\t%s\n" %
                         arc['l2_arc_breakdown']['value'])
        sys.stdout.write("\tHit Ratio:\t\t\t%s\t%s\n" % (
            arc['l2_arc_breakdown']['hit_ratio']['per'],
            arc['l2_arc_breakdown']['hit_ratio']['num'],
            )
        )

        sys.stdout.write("\tMiss Ratio:\t\t\t%s\t%s\n" % (
            arc['l2_arc_breakdown']['miss_ratio']['per'],
            arc['l2_arc_breakdown']['miss_ratio']['num'],
            )
        )

        sys.stdout.write("\tFeeds:\t\t\t\t\t%s\n" %
                         arc['l2_arc_breakdown']['feeds'])
        sys.stdout.write("\n")

        sys.stdout.write("L2 ARC Writes:\n")
        if arc['l2_writes_done'] != arc['l2_writes_sent']:
            sys.stdout.write("\tWrites Sent: (%s)\t\t\t\t%s\n" % (
                arc['l2_arc_writes']['writes_sent']['value'],
                arc['l2_arc_writes']['writes_sent']['num'],
                )
            )
            sys.stdout.write("\t  Done Ratio:\t\t\t%s\t%s\n" % (
                arc['l2_arc_writes']['done_ratio']['per'],
                arc['l2_arc_writes']['done_ratio']['num'],
                )
            )
            sys.stdout.write("\t  Error Ratio:\t\t\t%s\t%s\n" % (
                arc['l2_arc_writes']['error_ratio']['per'],
                arc['l2_arc_writes']['error_ratio']['num'],
                )
            )
        else:
            sys.stdout.write("\tWrites Sent:\t\t\t%s\t%s\n" % (
                arc['l2_arc_writes']['writes_sent']['per'],
                arc['l2_arc_writes']['writes_sent']['num'],
                )
            )


def get_dmu_summary(Kstat):
    """Collect information on the DMU"""

    output = {}

    zfetch_hits = Kstat["kstat.zfs.misc.zfetchstats.hits"]
    zfetch_misses = Kstat["kstat.zfs.misc.zfetchstats.misses"]

    zfetch_access_total = (zfetch_hits + zfetch_misses)
    output['zfetch_access_total'] = zfetch_access_total

    if zfetch_access_total > 0:
        output['dmu'] = {}
        output['dmu']['efficiency'] = {}
        output['dmu']['efficiency']['value'] = fHits(zfetch_access_total)
        output['dmu']['efficiency']['hit_ratio'] = {
            'per': fPerc(zfetch_hits, zfetch_access_total),
            'num': fHits(zfetch_hits),
        }
        output['dmu']['efficiency']['miss_ratio'] = {
            'per': fPerc(zfetch_misses, zfetch_access_total),
            'num': fHits(zfetch_misses),
        }

    return output


def _dmu_summary(Kstat):
    """Print information on the DMU"""

    arc = get_dmu_summary(Kstat)

    if arc['zfetch_access_total'] > 0:
        sys.stdout.write("DMU Prefetch Efficiency:\t\t\t\t\t%s\n" %
                         arc['dmu']['efficiency']['value'])
        sys.stdout.write("\tHit Ratio:\t\t\t%s\t%s\n" % (
            arc['dmu']['efficiency']['hit_ratio']['per'],
            arc['dmu']['efficiency']['hit_ratio']['num'],
            )
        )
        sys.stdout.write("\tMiss Ratio:\t\t\t%s\t%s\n" % (
            arc['dmu']['efficiency']['miss_ratio']['per'],
            arc['dmu']['efficiency']['miss_ratio']['num'],
            )
        )

        sys.stdout.write("\n")


def get_vdev_summary(Kstat):
    """Collect information on the VDEVs"""

    output = {}

    vdev_cache_delegations = \
        Kstat["kstat.zfs.misc.vdev_cache_stats.delegations"]
    vdev_cache_misses = Kstat["kstat.zfs.misc.vdev_cache_stats.misses"]
    vdev_cache_hits = Kstat["kstat.zfs.misc.vdev_cache_stats.hits"]
    vdev_cache_total = (vdev_cache_misses + vdev_cache_hits +
                        vdev_cache_delegations)

    output['vdev_cache_total'] = vdev_cache_total

    if vdev_cache_total > 0:
        output['summary'] = fHits(vdev_cache_total)
        output['hit_ratio'] = {
            'per': fPerc(vdev_cache_hits, vdev_cache_total),
            'num': fHits(vdev_cache_hits),
        }
        output['miss_ratio'] = {
            'per': fPerc(vdev_cache_misses, vdev_cache_total),
            'num': fHits(vdev_cache_misses),
        }
        output['delegations'] = {
            'per': fPerc(vdev_cache_delegations, vdev_cache_total),
            'num': fHits(vdev_cache_delegations),
        }

    return output


def _vdev_summary(Kstat):
    """Print information on the VDEVs"""

    arc = get_vdev_summary(Kstat)

    if arc['vdev_cache_total'] > 0:
        sys.stdout.write("VDEV Cache Summary:\t\t\t\t%s\n" % arc['summary'])
        sys.stdout.write("\tHit Ratio:\t\t\t%s\t%s\n" % (
            arc['hit_ratio']['per'],
            arc['hit_ratio']['num'],
        ))
        sys.stdout.write("\tMiss Ratio:\t\t\t%s\t%s\n" % (
            arc['miss_ratio']['per'],
            arc['miss_ratio']['num'],
        ))
        sys.stdout.write("\tDelegations:\t\t\t%s\t%s\n" % (
            arc['delegations']['per'],
            arc['delegations']['num'],
        ))


def _tunable_summary(Kstat):
    """Print information on tunables, including descriptions if requested"""

    global show_tunable_descriptions
    global alternate_tunable_layout

    tunables = load_tunables()
    descriptions = {}

    if show_tunable_descriptions:

        command = ["/sbin/modinfo", "zfs", "-0"]

        try:
            p = Popen(command, stdin=PIPE, stdout=PIPE,
                      stderr=PIPE, shell=False, close_fds=True)
            p.wait()

            # By default, Python 2 returns a string as the first element of the
            # tuple from p.communicate(), while Python 3 returns bytes which
            # must be decoded first. The better way to do this would be with
            # subprocess.run() or at least .check_output(), but this fails on
            # CentOS 6 because of its old version of Python 2
            desc = bytes.decode(p.communicate()[0])
            description_list = desc.strip().split('\0')

            if p.returncode == 0:
                for tunable in description_list:
                    if tunable[0:5] == 'parm:':
                        tunable = tunable[5:].strip()
                        name, description = tunable.split(':', 1)
                        if not description:
                            description = "Description unavailable"
                        descriptions[name] = description
            else:
                sys.stderr.write("%s: '%s' exited with code %i\n" %
                                 (sys.argv[0], command[0], p.returncode))
                sys.stderr.write("Tunable descriptions will be disabled.\n")
        except OSError as e:
            sys.stderr.write("%s: Cannot run '%s': %s\n" %
                             (sys.argv[0], command[0], e.strerror))
            sys.stderr.write("Tunable descriptions will be disabled.\n")

    sys.stdout.write("ZFS Tunables:\n")

    if alternate_tunable_layout:
        fmt = "\t%s=%s\n"
    else:
        fmt = "\t%-50s%s\n"

    for name in sorted(tunables.keys()):
        if show_tunable_descriptions and name in descriptions:
            sys.stdout.write("\t# %s\n" % descriptions[name])

        sys.stdout.write(fmt % (name, tunables[name]))


unSub = [
    _arc_summary,
    _arc_efficiency,
    _l2arc_summary,
    _dmu_summary,
    _vdev_summary,
    _tunable_summary
]


def zfs_header():
    """Print title string with date"""

    daydate = time.strftime('%a %b %d %H:%M:%S %Y')

    sys.stdout.write('\n'+'-'*72+'\n')
    sys.stdout.write('ZFS Subsystem Report\t\t\t\t%s' % daydate)
    sys.stdout.write('\n')


def usage():
    """Print usage information"""

    sys.stdout.write("Usage: arc_summary [-h] [-a] [-d] [-p PAGE]\n\n")
    sys.stdout.write("\t -h, --help           : "
                     "Print this help message and exit\n")
    sys.stdout.write("\t -a, --alternate      : "
                     "Show an alternate sysctl layout\n")
    sys.stdout.write("\t -d, --description    : "
                     "Show the sysctl descriptions\n")
    sys.stdout.write("\t -p PAGE, --page=PAGE : "
                     "Select a single output page to display,\n")
    sys.stdout.write("\t                        "
                     "should be an integer between 1 and " +
                     str(len(unSub)) + "\n\n")
    sys.stdout.write("Examples:\n")
    sys.stdout.write("\tarc_summary -a\n")
    sys.stdout.write("\tarc_summary -p 4\n")
    sys.stdout.write("\tarc_summary -ad\n")
    sys.stdout.write("\tarc_summary --page=2\n")


def main():
    """Main function"""

    global show_tunable_descriptions
    global alternate_tunable_layout

    try:
        opts, args = getopt.getopt(
            sys.argv[1:],
            "adp:h", ["alternate", "description", "page=", "help"]
        )
    except getopt.error as e:
        sys.stderr.write("Error: %s\n" % e.msg)
        usage()
        sys.exit(1)

    args = {}
    for opt, arg in opts:
        if opt in ('-a', '--alternate'):
            args['a'] = True
        if opt in ('-d', '--description'):
            args['d'] = True
        if opt in ('-p', '--page'):
            args['p'] = arg
        if opt in ('-h', '--help'):
            usage()
            sys.exit(0)

    Kstat = get_Kstat()

    alternate_tunable_layout = 'a' in args
    show_tunable_descriptions = 'd' in args

    pages = []

    if 'p' in args:
        try:
            pages.append(unSub[int(args['p']) - 1])
        except IndexError:
            sys.stderr.write('the argument to -p must be between 1 and ' +
                             str(len(unSub)) + '\n')
            sys.exit(1)
    else:
        pages = unSub

    zfs_header()
    for page in pages:
        page(Kstat)
        sys.stdout.write("\n")


if __name__ == '__main__':
    main()