#!/usr/bin/python3
# Run this with --help to see available options for tracing and debugging
# See https://github.com/cockpit-project/cockpit/blob/master/test/common/testlib.py
# "class Browser" and "class MachineCase" for the available API.

import re
import time

# import Cockpit's machinery for test VMs and its browser test API
import parent
import packagelib
from testlib import *


def getMaximumSpike(browser, g_type, saturation, hour, minute):
    # only for minutes with events, which have SVG graphs
    sel = "#metrics-hour-{0} div.metrics-data-{1}[style='--metrics-minute:{2};'] polygon".format(hour, g_type, minute)
    if saturation:
        sel += ":nth-child(2)"
    else:
        sel += ":first-child"

    points = browser.attr(sel, "points")
    assert "NaN" not in points, points
    xs = [float(x.split(",")[0]) for x in points.split()]

    return max(xs)


def getCompressedMinuteValue(browser, g_type, saturation, hour, minute):
    # only for minutes without events, which only have bars

    sel = "#metrics-hour-{0} div.metrics-data-{1}[style='--metrics-minute:{2};'] .compressed".format(hour, g_type, minute)
    m = re.search("--%s:([0-9.]+);" % (saturation and "saturation" or "utilization"), browser.attr(sel, "style"))
    assert m
    return float(m.group(1))


def progressValue(browser, progress_bar_sel):
    sel = progress_bar_sel + " .pf-c-progress__indicator"
    browser.wait_visible(sel)
    browser.wait_attr_contains(sel, "style", "width:")
    style = browser.attr(sel, "style")
    m = re.search("width: (\d+)%;", style)
    return int(m.group(1))


def topServiceValue(browser, aria_label, col_label, row):
    sel = "table[aria-label='%s'] tbody tr:nth-of-type(%d) td[data-label='%s']" % (aria_label, row, col_label)
    # split off unit, like "12 MB"
    return float(browser.text(sel).split(' ')[0])


def prepareArchive(machine, name, time):
    machine.upload(["verify/files/metrics-archives/{0}".format(name)], "/tmp/")
    machine.execute("""ntp=`timedatectl show --property NTP --value`
                       if [ $ntp == "yes" ]; then
                           timedatectl set-ntp off
                       fi
                       systemctl stop pmlogger
                       # don't let NM set transient host names from DHCP
                       systemctl stop NetworkManager
                       hostnamectl set-hostname localhost.localdomain
                       rm -rf /var/log/pcp/pmlogger/*
                       tar -C / -xzvf /tmp/{0}
                       # set-ntp off is asynchronous; wait until timesyncd stops before the time can be set
                       while systemctl is-active systemd-timesyncd; do sleep 1; done
                       timedatectl set-time @{1}""".format(name, time))

def login(self):
    # HACK: Ubuntu and Debian need some time until metrics channel is available
    # Really no idea what it needs to wait for, so let's just try channel until it succeeds
    if self.machine.image.startswith("ubuntu") or self.machine.image.startswith("debian"):
        self.login_and_go("/system")
        self.browser.wait(lambda: self.browser.call_js_func("""(function() {
            return new Promise((resolve, reject) => {
                cockpit.spawn(["date", "+%s"])
                    .then(out => {
                        const now = parseInt(out.trim()) * 1000;
                        const current_hour = Math.floor(now / 3600000) * 3600000;
                        const metrics_channel = cockpit.channel({ payload: "metrics1", source: "pcp-archive",
                            interval: 5000, metrics: [{ name: "kernel.all.cpu.nice", derive: "rate" }],
                            timestamp: current_hour, limit: 10 });
                        metrics_channel.addEventListener("close", (ev, error) => {
                            if (error.problem) {
                                console.log("Channel is not ready:", error.problem);
                                resolve(0);
                            } else
                                resolve(1);
                        });
                    });
                });
            })"""))
        self.browser.click("a:contains('View details and history')")
        self.browser.enter_page("/metrics")
    else:
        self.login_and_go("/metrics")


@skipImage("Do not test BaseOS packages", "rhel-8-3-distropkg", "rhel-8-4-distropkg")
class TestHistoryMetrics(MachineCase):
    def waitStream(self, current_max):
        # should only have at most <current_max> valid minutes, the rest should be empty
        valid_start = self.browser.call_js_func("ph_count", ".metrics-data-cpu.valid-data")
        self.assertLessEqual(valid_start, current_max)
        # page auto-updates every minute
        with self.browser.wait_timeout(90):
            self.browser.wait_js_func("(exp => ph_count('.metrics-data-cpu.valid-data') == exp)", valid_start + 1)

        # Should never show more then 4 empty leading minutes (block of 5 minutes but always at least one used)
        leading_empty = self.browser.call_js_func("""(function () {
            const lines = document.getElementsByClassName("metrics-data-cpu");
            let counter = 0;

            Array.from(lines).every(l => {
                if (l.classList.contains("empty-data")) {
                    counter++;
                    return true;
                } else {
                    return false;;
                }
            });

            return counter;
        })""")
        self.assertLessEqual(leading_empty, 4)

    @skipImage("no PCP support", "fedora-coreos")
    def testBasic(self):
        b = self.browser
        m = self.machine

        m.execute("systemctl stop pmlogger")
        m.execute("""ntp=`timedatectl show --property NTP --value`
             if [ $ntp == "yes" ]; then
                 timedatectl set-ntp off
             fi""")
        m.execute("while systemctl is-active systemd-timesyncd; do sleep 1; done")
        m.execute("timedatectl set-time '2020-11-24 09:24:05'")

        # clean slate, to avoid seeing the data from preparing the VM
        m.execute("rm -rf /var/log/pcp/pmlogger/*; systemctl start pmlogger")

        login(self)
        # eventually finishes data loading and shows heading
        b.wait_in_text(".metrics-history-heading", "CPU")

        # only shows current hour
        b.wait_js_func("ph_count_check", ".metrics-hour", 1)

        # VM just started, we don't have 12 hours of data
        b.wait_in_text(".metrics .pf-c-alert", "No data available between")
        # initial data gap is < 24 hours, does not show date
        year = m.execute("date +%Y").strip()
        self.assertNotIn(year, b.text(".metrics .pf-c-alert"))

        # can try to load earlier data; only updates "no data" alert as there is no data
        b.wait_text(".bottom-panel button", "Load earlier data")
        b.click(".bottom-panel button")
        # now the gap is > 24 hours, does show date
        b.wait_in_text(".metrics .pf-c-alert", year)
        # still only one hour
        b.wait_js_func("ph_count_check", ".metrics-hour", 1)

        self.waitStream(3)

        # Change date to yesterday, should be empty
        b.click(".pf-c-select__toggle-arrow")
        b.click(".pf-c-select__menu-wrapper:nth-child(2) button")
        b.wait_text(".pf-c-empty-state", "No data available")

        # Breadcrumb back to Overview page
        b.click(".pf-c-breadcrumb li:first-child a")
        b.enter_page("/system")
        b.wait_visible('.system-information')

    @skipImage("no PCP support", "fedora-coreos")
    def testEvents(self):
        b = self.browser
        m = self.machine

        def events_at(hour, minute):
            return b.text("#metrics-hour-{0} .metrics-events[style='--metrics-minute:{1};']".format(hour, minute))

        #
        # Disks
        #

        # disable swap, so that we can test current metrics without swap
        m.execute('''systemctl stop "*.swap" "swap-create@*" || true
                     systemctl mask "swap-create@"
                     swapoff --all
                     while [ -n "$(swapon --show)" ]; do sleep 1; done''')

        prepareArchive(m, "disk.tar.gz", 1597672800)

        login(self)
        # eventually finishes data loading and shows heading
        b.wait_in_text(".metrics-history-heading", "CPU")

        # Big spike lasting 3 minutes
        self.assertGreaterEqual(getMaximumSpike(b, "disks", False, 1597662000000, 25), 0.9)
        self.assertGreaterEqual(getCompressedMinuteValue(b, "disks", False, 1597662000000, 26), 0.9)
        self.assertGreaterEqual(getCompressedMinuteValue(b, "disks", False, 1597662000000, 27), 0.9)

        # Smaller spike lasting 2 minutes
        self.assertGreaterEqual(getMaximumSpike(b, "disks", False, 1597662000000, 28), 0.4)
        self.assertLessEqual(getMaximumSpike(b, "disks", False, 1597662000000, 28), 0.6)
        self.assertGreaterEqual(getCompressedMinuteValue(b, "disks", False, 1597662000000, 29), 0.4)
        # recognized as event
        self.assertIn("Disk I/O spike", events_at(1597662000000, 28))

        # No visible activity after that
        self.assertLessEqual(getCompressedMinuteValue(b, "disks", False, 1597662000000, 30), 0.01)

        # swap usage is not shown if there is no swap
        b.wait_visible("#current-memory-usage")
        self.assertFalse(b.is_present("#current-swap-usage"))

        # Check that we don't show too much empty minutes in the first hour
        self.assertLessEqual(b.call_js_func("ph_count", ".metrics-data-cpu"), 35)

        b.logout()

        #
        # Network and CPU
        #

        prepareArchive(m, "cpu_network.tar.gz", 1598918400)

        login(self)
        # eventually finishes data loading and shows heading
        b.wait_in_text(".metrics-history-heading", "CPU")

        # Test network - Big spike lasting 2 minutes
        self.assertGreaterEqual(getMaximumSpike(b, "network", False, 1598950800000, 3), 0.5)
        self.assertGreaterEqual(getMaximumSpike(b, "network", False, 1598950800000, 4), 0.5)
        # recognized as event
        self.assertIn("Network I/O spike", events_at(1598950800000, 3))
        # but it's not a new event in minute 4
        self.assertNotIn("Network I/O spike", events_at(1598950800000, 4))

        # Followed by smaller spike
        self.assertGreaterEqual(getMaximumSpike(b, "network", False, 1598950800000, 5), 0.35)
        self.assertLessEqual(getMaximumSpike(b, "network", False, 1598950800000, 5), 0.5)
        # still not a new spike
        self.assertNotIn("Network I/O spike", events_at(1598950800000, 5))

        # Followed by virtually no data
        self.assertLessEqual(getCompressedMinuteValue(b, "network", False, 1598950800000, 6), 0.01)

        # Test CPU load - big - small - big spikes
        self.assertGreaterEqual(getMaximumSpike(b, "cpu", False, 1598950800000, 3), 0.9)
        self.assertGreaterEqual(getMaximumSpike(b, "cpu", False, 1598950800000, 4), 0.5)
        self.assertLessEqual(getMaximumSpike(b, "cpu", False, 1598950800000, 4), 0.55)
        self.assertGreaterEqual(getMaximumSpike(b, "cpu", False, 1598950800000, 5), 0.9)
        self.assertIn("CPU spike", events_at(1598950800000, 2))
        self.assertIn("CPU spike", events_at(1598950800000, 5))

        # Test CPU saturation - 3 spikes, each 2 minutes (medium, big, small)
        self.assertGreaterEqual(getMaximumSpike(b, "cpu", True, 1598950800000, 3), 0.5)
        self.assertLessEqual(getMaximumSpike(b, "cpu", True, 1598950800000, 3), 0.6)
        self.assertGreaterEqual(getMaximumSpike(b, "cpu", True, 1598950800000, 4), 0.5)
        self.assertLessEqual(getMaximumSpike(b, "cpu", True, 1598950800000, 4), 0.6)

        self.assertGreaterEqual(getMaximumSpike(b, "cpu", True, 1598950800000, 5), 0.8)
        self.assertGreaterEqual(getCompressedMinuteValue(b, "cpu", True, 1598950800000, 6), 0.8)

        self.assertGreaterEqual(getCompressedMinuteValue(b, "cpu", True, 1598950800000, 7), 0.3)
        self.assertLessEqual(getCompressedMinuteValue(b, "cpu", True, 1598950800000, 7), 0.4)
        self.assertGreaterEqual(getCompressedMinuteValue(b, "cpu", True, 1598950800000, 8), 0.3)
        self.assertLessEqual(getCompressedMinuteValue(b, "cpu", True, 1598950800000, 8), 0.4)

        self.assertNotIn("Load spike", events_at(1598950800000, 2))
        self.assertIn("Load spike", events_at(1598950800000, 3))
        self.assertNotIn("Load spike", events_at(1598950800000, 4))
        self.assertIn("Load spike", events_at(1598950800000, 5))

        b.logout()

        #
        # Memory
        #

        prepareArchive(m, "memory.tar.gz", 1600248000)
        login(self)
        b.wait_in_text(".metrics-history-heading", "CPU")

        # basic RAM consumption after boot; it's still a network spike, thus event+SVG
        self.assertLessEqual(getMaximumSpike(b, "memory", False, 1600236000000, 44), 0.3)
        self.assertAlmostEqual(getMaximumSpike(b, "memory", True, 1600236000000, 44), 0)
        self.assertNotIn("Memory spike", events_at(1600236000000, 44))
        self.assertNotIn("Swap", events_at(1600236000000, 44))

        # swap event from :46 to :47
        self.assertGreater(getMaximumSpike(b, "memory", True, 1600236000000, 46), 0.9)
        self.assertIn("Swap", events_at(1600236000000, 46))
        # continuous, no new Swap event, but still a Memory+Network event
        self.assertGreater(getMaximumSpike(b, "memory", True, 1600236000000, 47), 0.9)
        self.assertNotIn("Swap", events_at(1600236000000, 47))

        # memory spike in :47
        self.assertGreater(getMaximumSpike(b, "memory", False, 1600236000000, 47), 0.6)
        self.assertIn("Memory spike", events_at(1600236000000, 47))

        # bigger memory spike in :53
        self.assertGreater(getMaximumSpike(b, "memory", False, 1600236000000, 53), 0.8)
        self.assertIn("Memory spike", events_at(1600236000000, 53))

        # continued memory spike in :54, but no new events
        self.assertGreater(getCompressedMinuteValue(b, "memory", False, 1600236000000, 54), 0.8)
        self.assertAlmostEqual(getCompressedMinuteValue(b, "memory", True, 1600236000000, 54), 0.0)

        # everything is quiet in :55
        self.assertLess(getCompressedMinuteValue(b, "memory", False, 1600236000000, 55), 0.4)
        self.assertAlmostEqual(getCompressedMinuteValue(b, "memory", True, 1600236000000, 55), 0.0)

        b.logout()

        #
        # Check changing of time
        #

        m.execute("timedatectl set-time @1600550674")
        login(self)
        # self.waitStream(3) # FIXME: wait for new data - pcp does not handle time change greatly
        b.wait_text(".pf-c-select__toggle-text", "Today")

        b.select_PF4("#date-picker-select-toggle", "Wednesday, Sep 16, 2020")
        self.assertGreater(getMaximumSpike(b, "memory", False, 1600236000000, 53), 0.8)
        self.assertIn("Memory spike", events_at(1600236000000, 53))

        b.click(".pf-c-select__toggle")
        b.click(".pf-c-select__menu-item:contains('Today')")
        b.wait_text(".pf-c-select__toggle-text", "Today")
        # self.waitStream(4) # FIXME: wait for new data - pcp does not handle time change greatly

    @nondestructive
    @skipImage("no PCP support", "fedora-coreos")
    def testNoData(self):
        b = self.browser
        m = self.machine

        m.execute("systemctl stop pmlogger && mount -t tmpfs tmpfs /var/log/pcp/pmlogger")
        self.addCleanup(m.execute, "umount /var/log/pcp/pmlogger")

        self.login_and_go("/metrics")

        b.wait_in_text(".pf-c-empty-state", "Metrics history could not be loaded")
        # Troubleshoot
        b.click(".pf-c-empty-state button.pf-m-link")
        b.enter_page("/system/services")
        b.wait_in_text("#service-details", "pmlogger.service")


@skipImage("Do not test BaseOS packages", "rhel-8-3-distropkg", "rhel-8-4-distropkg")
@nondestructive
class TestCurrentMetrics(MachineCase):
    def setUp(self):
        super().setUp()
        # packagekit/dnf often eats a lot of CPU; silence it to have better control over CPU usage
        self.machine.execute("systemctl mask packagekit && killall -9 /usr/libexec/packagekitd && killall -9 dnf || true")
        self.addCleanup(self.machine.execute, "systemctl unmask packagekit")
        # make sure to clean up our test resource consumers on failures
        self.addCleanup(self.machine.execute, "systemctl stop cockpittest.slice 2>/dev/null || true")
        login(self)

    def testCPU(self):
        b = self.browser
        m = self.machine

        nproc = m.execute("nproc").strip()
        b.wait_in_text("#current-cpu-usage", nproc + " CPU")
        # wait until system settles down
        b.wait(lambda: progressValue(b, "#current-cpu-usage") < 20)
        m.execute("systemd-run --collect --slice cockpittest -p CPUQuota=60% --unit cpu-hog dd if=/dev/urandom of=/dev/null")
        m.execute("systemd-run --collect --slice cockpittest -p CPUQuota=30% --unit cpu-piglet dd if=/dev/urandom of=/dev/null")
        b.wait(lambda: progressValue(b, "#current-cpu-usage") > 75)
        # no other process in the test VM should take > 30% CPU, by the "settles down" assertion above
        b.wait_text("table[aria-label='Top 5 CPU services'] tbody tr:nth-of-type(1) td[data-label='Service']", "cpu-hog")
        b.wait_text("table[aria-label='Top 5 CPU services'] tbody tr:nth-of-type(2) td[data-label='Service']", "cpu-piglet")

        # There might be some other processes which take more resources
        # Keep this logging so we can easily debug which ones we might need to cleanup
        try:
            b.wait(lambda: topServiceValue(b, "Top 5 CPU services", "%", 1) > 50)
            b.wait(lambda: topServiceValue(b, "Top 5 CPU services", "%", 1) < 70)
            b.wait(lambda: topServiceValue(b, "Top 5 CPU services", "%", 2) > 20)
            b.wait(lambda: topServiceValue(b, "Top 5 CPU services", "%", 2) < 40)
        except:
            print(m.execute("top -b -n 1"))
            raise

        m.execute("systemctl stop cpu-hog cpu-piglet")
        # should go back to idle usage
        b.wait(lambda: progressValue(b, "#current-cpu-usage") < 20)
        # it could be that the table disappears completely if no service has a noticeable CPU usage;
        # so don't assume the table exists
        b.wait_not_in_text("#current-metrics-card-cpu", "cpu-hog")
        b.wait_not_in_text("#current-metrics-card-cpu", "cpu-piglet")

        # Load looks like "1 min: 1.41, 5 min: 1.47, 15 min: 2.30"
        b.wait(lambda: float(b.text("#load-avg").split()[2].rstrip(',')) < 5)

        # HACK: debian-stable does not update load when done though systemd-run
        if m.image != "debian-stable":
            m.execute("systemd-run --collect --slice cockpittest --unit load-hog sh -ec "
                      "  'for i in `seq 500`; do dd if=/dev/urandom of=/dev/zero bs=10K count=500 status=none & done'")
            b.wait(lambda: float(b.text("#load-avg").split()[2].rstrip(',')) > 15)
            m.execute("systemctl stop load-hog 2>/dev/null || true") # ok to fail, as the command exits by itself
        else:
            load_hog = m.spawn("for i in `seq 500`; do dd if=/dev/urandom of=/dev/zero bs=10K count=500 status=none & done", "load-hog.log")
            b.wait(lambda: float(b.text("#load-avg").split()[2].rstrip(',')) > 15)
            m.execute("kill %d 2>/dev/null || true" % load_hog) # ok to fail, as the command exits by itself

        # this settles down slowly, don't wait for becoming really quiet
        with b.wait_timeout(180):
            b.wait(lambda: float(b.text("#load-avg").split()[2].rstrip(',')) < 10)

    def testMemory(self):
        b = self.browser
        m = self.machine
        # only some images have swap
        have_swap = m.execute("swapon --show").strip()
        # wait until RAM usage is initialized
        b.wait(lambda: progressValue(b, "#current-memory-usage") > 10)

        # our test machines have ~ 1 GiB of memory, a reasonable chunk of it should be used
        b.wait_in_text("#current-memory-usage", " GiB")
        initial_usage = progressValue(b, "#current-memory-usage")
        self.assertGreater(initial_usage, 10)
        self.assertLess(initial_usage, 80)
        # allocate a chunk of memory; this may cause other stuff to get unmapped,
        # thus not exact addition, but usage should go up
        size = 300 if have_swap else 200
        m.execute("systemd-run --collect --slice cockpittest --unit mem-hog sh -ec "
                  f"'MEMBLOB=$(yes | dd bs=1M count={size} iflag=fullblock); touch /tmp/hogged; sleep infinity'")
        m.execute("while [ ! -e /tmp/hogged ]; do sleep 1; done")
        # bars update every 3s
        time.sleep(8)
        hog_usage = progressValue(b, "#current-memory-usage")
        self.assertGreater(hog_usage, initial_usage + 8)

        b.wait_text("table[aria-label='Top 5 memory services'] tbody tr:nth-of-type(1) td[data-label='Service']", "mem-hog")
        b.wait(lambda: topServiceValue(b, "Top 5 memory services", "Used", 1) > size)
        b.wait(lambda: topServiceValue(b, "Top 5 memory services", "Used", 1) < size + 50)

        # total memory is shown as tooltip
        b.mouse("#current-memory-usage", "mouseenter")
        b.wait_in_text(".pf-c-tooltip", "GiB total")
        b.mouse("#current-memory-usage", "mouseleave")

        # table entries are links to Services page
        b.click("table[aria-label='Top 5 memory services'] tbody tr:nth-of-type(1) td[data-label='Service'] a span")
        b.enter_page("/system/services")
        b.wait_in_text("#path", "/mem-hog.service")
        b.wait_in_text(".service-name", "MEMBLOB=")

        b.go("/metrics")
        b.enter_page("/metrics")
        b.wait_visible("table[aria-label='Top 5 memory services']")

        if have_swap:
            # use even more memory to trigger swap
            m.execute("systemd-run --collect --slice cockpittest --unit mem-hog2 sh -ec 'MEMBLOB=$(yes | dd bs=1M count=300 iflag=fullblock); sleep infinity'")
            b.wait(lambda: progressValue(b, "#current-swap-usage") > 0)

            m.execute("systemctl stop mem-hog mem-hog2")

            # should go back to initial_usage; often below, due to paged out stuff
            b.wait(lambda: progressValue(b, "#current-memory-usage") <= initial_usage)
            self.assertGreater(progressValue(b, "#current-memory-usage"), 10)
            b.wait_not_in_text("table[aria-label='Top 5 memory services'] tbody", "mem-hog")

            # total swap is shown as tooltip
            b.mouse("#current-swap-usage", "mouseenter")
            b.wait_in_text(".pf-c-tooltip", "GiB total")
            b.mouse("#current-swap-usage", "mouseleave")

    def testDiskIO(self):
        b = self.browser
        m = self.machine
        login(self)

        # test env should be quiet enough to not transmit MB/s
        b.wait(lambda: re.match(r'^(0|[0-9.]+ (KiB|B)/s)$', b.text("#current-disks-read")))
        b.wait(lambda: re.match(r'^(0|[0-9.]+ (KiB|B)/s)$', b.text("#current-disks-write")))
        # reading lots of data
        m.execute("systemd-run --collect --slice cockpittest --unit disk-read-hog sh -ec 'while true; do echo 3 > /proc/sys/vm/drop_caches; grep -r . /usr >/dev/null; done'")
        b.wait(lambda: re.match(r'^[0-9.]+ (MiB|GiB)/s$', b.text("#current-disks-read")))
        b.wait(lambda: re.match(r'^[0-9.]+ (KiB|B)/s$', b.text("#current-disks-write")))  # this should stay calm
        m.execute("systemctl stop disk-read-hog")
        b.wait(lambda: re.match(r'^[0-9.]+ (KiB|B)/s$', b.text("#current-disks-read")))  # back to quiet
        # writing lots of data
        m.execute("systemd-run --collect --slice cockpittest --unit disk-write-hog sh -ec "
                  " 'while true; do dd if=/dev/zero of=/var/tmp/blob bs=1M count=100; done'")
        self.addCleanup(m.execute, "rm -f /var/tmp/blob")
        b.wait(lambda: re.match(r'^[0-9.]+ (MiB|GiB)/s$', b.text("#current-disks-write")))
        b.wait(lambda: re.match(r'^[0-9.]+ (KiB|B)/s$', b.text("#current-disks-read")))  # this should stay calm
        m.execute("systemctl stop disk-write-hog")
        b.wait(lambda: re.match(r'^(0|[0-9.]+ (KiB|B)/s)$', b.text("#current-disks-write")))

        # Disk usage

        # add 50 MB loopback disk
        m.execute("""set -e
                  F=$(mktemp /var/tmp/loop.XXXX)
                  dd if=/dev/zero of=$F bs=1M count=50
                  mkfs -t ext3 $F
                  mkdir -p /var/cockpittest
                  mount -o loop $F /var/cockpittest
                  rm $F
                  """)
        self.addCleanup(m.execute, "umount /var/cockpittest")
        self.assertLess(progressValue(b, ".pf-c-progress[data-disk-usage-target='/var/cockpittest']"), 5)
        progress_sel = ".pf-c-progress[data-disk-usage-target='/var/cockpittest'] .pf-c-progress__status"
        # free size is anything between 40 and 50 MB
        self.assertRegex(b.text(progress_sel), "^4\d\.\d MB free$")
        # total size is shown in tooltip
        b.mouse(progress_sel, "mouseenter")
        b.wait_in_text(".pf-c-tooltip", "total")
        # total size is anything between 40 and 50 MB
        self.assertRegex(b.text(".pf-c-tooltip"), "^4\d\.\d MB total$")
        b.mouse(progress_sel, "mouseleave")

        m.execute("dd if=/dev/zero of=/var/cockpittest/blob bs=1M count=40")
        b.wait(lambda: progressValue(b, ".pf-c-progress[data-disk-usage-target='/var/cockpittest']") >= 90)

        # clicking on progress leads to the storage page
        if m.image != "fedora-coreos":
            self.assertTrue(b.is_present("#current-disks-usage button"))
            b.click(progress_sel)
            b.enter_page("/storage")
            # weird -- storage page does not show transient mount points, only permanent ones; so check for the device
            dev = m.execute("findmnt --noheadings -o SOURCE /var/cockpittest").strip()
            b.wait_in_text("#mounts", dev)

            b.go("/metrics")
            b.enter_page("/metrics")
            b.wait_visible(progress_sel)
            b.logout()

            # without cockpit-storaged, mounts are not links
            self.restore_file("/usr/share/cockpit/storaged/manifest.json")
            m.write("/usr/share/cockpit/storaged/manifest.json", "")
            self.allow_journal_messages("storaged: couldn't read manifest.json: JSON data was empty")
            login(self)
        b.wait_visible(progress_sel)
        self.assertFalse(b.is_present("#current-disks-usage button"))

    def testNetwork(self):
        b = self.browser
        m = self.machine

        # add synthetic veth which is guaranteed quiet
        m.execute("ip link add name cockpittest1 type veth peer name vcockpittest1")
        self.addCleanup(m.execute, "ip link del dev cockpittest1")

        # has expected interfaces
        b.wait_in_text("[aria-label='Network usage'] [data-interface='cockpittest1']", "cockpittest1")
        b.wait_in_text("[aria-label='Network usage'] [data-interface='lo']", "lo")

        def rateMatches(label, regexp):
            text = b.text("[aria-label='Network usage'] [data-interface='lo'] td[data-label='%s']" % label)
            return re.match(regexp, text) is not None

        # loopback is quiet enough to not transmit MB/s
        b.wait(lambda: rateMatches("In", r'^(0|[0-9.]+ (KiB|B)/s)$'))
        b.wait(lambda: rateMatches("Out", r'^(0|[0-9.]+ (KiB|B)/s)$'))
        # pipe lots of data through lo
        m.execute("systemd-run --collect --slice cockpittest --unit lo-hog sh -ec "
                  " 'nc -n -vv -l 2000 > /dev/null & sleep 1; nc -vv localhost 2000 </dev/zero'")
        b.wait(lambda: rateMatches("In", r'^[0-9.]+ (MiB|GiB)/s$'))
        b.wait(lambda: rateMatches("Out", r'^[0-9.]+ (MiB|GiB)/s$'))
        m.execute("systemctl stop lo-hog")

        # nothing happens on cockpittest1
        b.wait_text("[aria-label='Network usage'] [data-interface='cockpittest1'] td[data-label='In']", "0")
        b.wait_text("[aria-label='Network usage'] [data-interface='cockpittest1'] td[data-label='Out']", "0")


@skipImage("Do not test BaseOS packages", "rhel-8-3-distropkg", "rhel-8-4-distropkg")
class TestCockpitPcp(packagelib.PackageCase):
    def testNoCockpitPcp(self):
        b = self.browser
        m = self.machine

        if m.image in ["fedora-coreos"]:
            self.login_and_go("/metrics")
            b.wait_in_text(".pf-c-empty-state", "cockpit-pcp is missing")
            b.wait_not_present(".pf-c-empty-state button.pf-m-primary")
            return

        m.execute("mount -t tmpfs tmpfs /usr/share/cockpit/pcp")
        self.addCleanup(m.execute, "umount /usr/share/cockpit/pcp")

        m.execute("pkcon remove -y pcp")

        cpcp = {
            "/usr/share/cockpit/pcp/manifest.json": '{"requires": {"cockpit": "134.x"}, "bridges": [{"match": { "payload": "metrics1"},"spawn": [ "/usr/libexec/cockpit-pcp" ]}]}',
            "/usr/libexec/cockpit-pcp": "true",
        }

        self.createPackage("cockpit-pcp", "999", "1", content=cpcp,
                           postinst="chmod +x /usr/libexec/cockpit-pcp")
        self.enableRepo()
        m.execute("pkcon refresh")

        self.login_and_go("/metrics")
        b.wait_in_text(".pf-c-empty-state", "cockpit-pcp is missing")
        b.click(".pf-c-empty-state button.pf-m-primary")
        b.click("#dialog button:contains('Install')")
        b.wait_not_present("#dialog")
        b.click("button:contains('Log out')")
        b.leave_page()
        b.click("button:contains('Reconnect')")
        b.expect_load()
        b.set_val("#login-user-input", "admin")
        b.set_val("#login-password-input", "foobar")
        b.click('#login-button')
        b.expect_load()
        b.enter_page("/metrics")
        b.wait_in_text(".pf-c-empty-state", "Metrics history could not be loaded")


@skipImage("Do not test BaseOS packages", "rhel-8-3-distropkg", "rhel-8-4-distropkg")
class TestMultiCPU(MachineCase):

    provision = {
        "0": {"cpus": 2}
    }

    @skipImage("no PCP support", "fedora-coreos")
    def testCPUUsage(self):
        b = self.browser
        m = self.machine

        prepareArchive(m, "2corescpu.tar.gz", 1598971635)
        login(self)

        # one core is busy, the other idle -- that should be 50% total usage
        self.assertGreaterEqual(getCompressedMinuteValue(b, "cpu", False, 1598968800000, 44), 0.2)
        self.assertLessEqual(getCompressedMinuteValue(b, "cpu", False, 1598968800000, 44), 0.55)

        # next minute, both cores are busy
        self.assertGreaterEqual(getMaximumSpike(b, "cpu", False, 1598968800000, 45), 0.5)
        self.assertLessEqual(getMaximumSpike(b, "cpu", False, 1598968800000, 45), 1.0)

        # Test current usage of cores
        b.wait(lambda: progressValue(b, "#current-cpu-usage") < 20)
        m.execute("systemd-run --collect --slice cockpittest -p CPUQuota=60% --unit cpu-hog dd if=/dev/urandom of=/dev/null")
        m.execute("systemd-run --collect --slice cockpittest -p CPUQuota=30% --unit cpu-piglet dd if=/dev/urandom of=/dev/null")
        b.wait(lambda: progressValue(b, "#current-cpu-usage") > 40)
        b.mouse("#current-cpu-usage", "mouseenter")
        b.wait(lambda: int(b.text(".pf-c-tooltip .pf-l-flex:first-child div:nth-child(2)")[:-1]) > 50)
        b.wait(lambda: int(b.text(".pf-c-tooltip .pf-l-flex:nth-child(2) div:nth-child(2)")[:-1]) > 20)


if __name__ == '__main__':
    test_main()
