Implemented collector utility.

plugins/cpu_plugin.py

@@ -0,0 +1,53 @@
+from datetime import datetime
+
+import psutil
+from peewee import *
+from playhouse.shortcuts import model_to_dict
+
+import config
+from database import BaseModel
+
+from .plugin import Plugin
+
+
+class Cpu(BaseModel):
+    time = DateTimeField(index=True, default=datetime.now)
+    cpu = SmallIntegerField(null=True)
+    idle_pct = FloatField(null=False)
+    user_pct = FloatField(null=False)
+    system_pct = FloatField(null=False)
+    nice_pct = FloatField(null=True)
+    iowait_pct = FloatField(null=True)
+    irq_pct = FloatField(null=True)
+    softirq_pct = FloatField(null=True)
+    freq_min = FloatField(null=True)
+    freq_current = FloatField(null=True)
+    freq_max = FloatField(null=True)
+
+
+class CpuPlugin(Plugin):
+    models = [Cpu]
+
+    def store(self, cpu, times, freq):
+        entry = Cpu()
+        entry.cpu = cpu
+        entry.idle_pct = times.idle
+        entry.user_pct = times.user
+        entry.system_pct = times.system
+        entry.nice_pct = getattr(times, 'nice', None)
+        entry.iowait_pct = getattr(times, 'iowait', None)
+        entry.irq_pct = getattr(times, 'irq', getattr(times, 'interrupt', None))
+        entry.softirq_pct = getattr(times, 'softirq', None)
+        entry.freq_min = getattr(freq, 'min', None)
+        entry.freq_current = getattr(freq, 'current', None)
+        entry.freq_max = getattr(freq, 'max', None)
+        entry.save()
+
+    def execute(self):
+        self.store(None, psutil.cpu_times_percent(percpu=False), psutil.cpu_freq(percpu=False))
+
+        if config.CPU_PER_CPU:
+            times = psutil.cpu_times_percent(percpu=True)
+            freqs = psutil.cpu_freq(percpu=True)
+            for i in range(len(times)):
+                self.store(i, times[i], freqs[i])

plugins/disk_plugin.py

@@ -0,0 +1,74 @@
+from collections import namedtuple
+from datetime import datetime
+
+import psutil
+from peewee import *
+from playhouse.shortcuts import model_to_dict
+
+import config
+from database import BaseModel
+
+from .plugin import Plugin
+
+
+class DiskUsage(BaseModel):
+    time = DateTimeField(index=True, default=datetime.now)
+    partition = TextField(null=False)
+    mountpoint = TextField(null=False)
+    total = BigIntegerField(null=False)
+    used = BigIntegerField(null=False)
+    free = BigIntegerField(null=False)
+
+
+class DiskIO(BaseModel):
+    time = DateTimeField(index=True, default=datetime.now)
+    disk = TextField(null=True)
+    read_count = FloatField(null=False)     # all values are per second
+    write_count = FloatField(null=False)
+    read_speed = FloatField(null=False)
+    write_speed = FloatField(null=False)
+
+
+class DiskPlugin(Plugin):
+    models = [DiskUsage, DiskIO]
+
+    def __init__(self):
+        self.__i = 0
+        self.__previous_io = {}
+
+    def store_io(self, disk, current):
+        previous = self.__previous_io.get(disk, current)
+
+        entry = DiskIO()
+        entry.disk = disk
+        entry.read_count = (current.read_count - previous.read_count) / config.INTERVAL
+        entry.write_count = (current.write_count - previous.write_count) / config.INTERVAL
+        entry.read_speed = (current.read_bytes - previous.read_bytes) / config.INTERVAL
+        entry.write_speed = (current.write_bytes - previous.write_bytes) / config.INTERVAL
+        entry.save()
+
+        self.__previous_io[disk] = current
+
+    def execute(self):
+
+        # Collect disk usage
+        if (self.__i % config.DISK_SPACE_FREQUENCY) == 0:
+            for partition in psutil.disk_partitions():
+                usage = psutil.disk_usage(partition.mountpoint)
+                
+                entry = DiskUsage()
+                entry.partition = partition.device
+                entry.mountpoint = partition.mountpoint
+                entry.total = usage.total
+                entry.used = usage.used
+                entry.free = usage.free
+                entry.save()
+
+        # Collect IO
+        self.store_io(None, psutil.disk_io_counters(perdisk=False))
+
+        io_reads = psutil.disk_io_counters(perdisk=True)
+        for disk, current in io_reads.items():
+            self.store_io(disk, current)
+
+        self.__i += 1

plugins/memory_plugin.py

@@ -0,0 +1,46 @@
+from datetime import datetime
+
+import psutil
+from peewee import *
+
+import config
+from database import BaseModel
+
+from .plugin import Plugin
+
+
+class Memory(BaseModel):
+    time = DateTimeField(index=True, default=datetime.now)
+    total = BigIntegerField(null=False)
+    available = BigIntegerField(null=False)
+    used = BigIntegerField(null=False)
+    free = BigIntegerField(null=False)
+    active = BigIntegerField(null=True)
+    inactive = BigIntegerField(null=True)
+    buffers = BigIntegerField(null=True)
+    cached = BigIntegerField(null=True)
+    swap_total = BigIntegerField(null=False)
+    swap_used = BigIntegerField(null=False)
+    swap_free = BigIntegerField(null=False)
+
+
+class MemoryPlugin(Plugin):
+    models = [Memory]
+
+    def execute(self):
+        vmem = psutil.virtual_memory()
+        swap = psutil.swap_memory()
+
+        entry = Memory()
+        entry.total = vmem.total
+        entry.available = vmem.available
+        entry.used = vmem.used
+        entry.free = vmem.free
+        entry.active = getattr(vmem, 'active', None)
+        entry.inactive = getattr(vmem, 'inactive', None)
+        entry.buffers = getattr(vmem, 'buffers', None)
+        entry.cached = getattr(vmem, 'cached', None)
+        entry.swap_total = swap.total
+        entry.swap_free = swap.free
+        entry.swap_used = swap.used
+        entry.save()

plugins/network_plugin.py

@@ -0,0 +1,48 @@
+from collections import namedtuple
+from datetime import datetime
+
+import psutil
+from peewee import *
+from playhouse.shortcuts import model_to_dict
+
+import config
+from database import BaseModel
+
+from .plugin import Plugin
+
+
+class NetworkIO(BaseModel):
+    time = DateTimeField(index=True, default=datetime.now)
+    nic = TextField(null=True)
+    packets_sent = FloatField(null=False)     # all values are per second
+    packets_recv = FloatField(null=False)
+    bytes_sent = FloatField(null=False)
+    bytes_recv = FloatField(null=False)
+
+
+class NetworkPlugin(Plugin):
+    models = [NetworkIO]
+
+    def __init__(self):
+        self.__previous_io = {}
+
+    def store_io(self, nic, current):
+        previous = self.__previous_io.get(nic, current)
+
+        entry = NetworkIO()
+        entry.nic = nic
+        entry.packets_sent = (current.packets_sent - previous.packets_sent) / config.INTERVAL
+        entry.packets_recv = (current.packets_recv - previous.packets_recv) / config.INTERVAL
+        entry.bytes_sent = (current.bytes_sent - previous.bytes_sent) / config.INTERVAL
+        entry.bytes_recv = (current.bytes_recv - previous.bytes_recv) / config.INTERVAL
+        entry.save()
+
+        self.__previous_io[nic] = current
+
+    def execute(self):
+
+        self.store_io(None, psutil.net_io_counters(pernic=False))
+
+        io_reads = psutil.net_io_counters(pernic=True)
+        for nic, current in io_reads.items():
+            self.store_io(nic, current)

plugins/ping_plugin.py

@@ -0,0 +1,61 @@
+import asyncio
+import re
+import subprocess
+from datetime import datetime
+
+import psutil
+from peewee import *
+from playhouse.shortcuts import model_to_dict
+
+import config
+from database import BaseModel
+
+from .plugin import Plugin
+
+
+class Ping(BaseModel):
+    time = DateTimeField(index=True, default=datetime.now)
+    host = TextField(null=False)
+    ping = FloatField(null=True) # null = timeout or error
+
+
+class PingPlugin(Plugin):
+    models = [Ping]
+
+    def __init__(self):
+        self.__timeout = config.INTERVAL // 3
+        self.__i = 0
+
+    async def do_ping(self, host):
+        command = ['ping', '-c', '1', '-W', str(self.__timeout), host]
+        proc = await asyncio.create_subprocess_shell(' '.join(command), 
+                                                     stdout=asyncio.subprocess.PIPE)
+
+        stdout,_ = await proc.communicate()
+        stdout = stdout.decode()
+
+        entry = Ping()
+        entry.host = host
+        entry.ping = None
+
+        match = re.search(r'time=([\d\.]+) ms', stdout)
+        if match is not None:
+            entry.ping = float(match.group(1))
+
+        entry.save()
+        print(model_to_dict(entry))
+
+    async def execute_internal(self):
+        await asyncio.gather(*[self.do_ping(host) for host in config.PING_HOSTS])
+
+    def execute(self):
+        if (self.__i % config.PING_FREQUENCY) == 0:
+            if getattr(asyncio, 'run', None) is not None:
+                # Python 3.7+
+                asyncio.run(self.execute_internal())
+            else:
+                loop = asyncio.get_event_loop()
+                loop.run_until_complete(self.execute_internal())
+                loop.close()
+        
+        self.__i += 1

plugins/plugin.py

@@ -0,0 +1,10 @@
+from abc import ABC, abstractmethod
+from typing import Tuple
+
+
+class Plugin(ABC):
+    models = []
+
+    @abstractmethod
+    def execute(self) -> None:
+        pass

plugins/temperatures_plugin.py

@@ -0,0 +1,35 @@
+from collections import namedtuple
+from datetime import datetime
+
+import psutil
+from peewee import *
+from playhouse.shortcuts import model_to_dict
+
+import config
+from database import BaseModel
+
+from .plugin import Plugin
+
+
+class Temperatures(BaseModel):
+    time = DateTimeField(index=True, default=datetime.now)
+    sensor = TextField(null=False)
+    sensor_label = TextField(null=False)
+    current = FloatField(null=False)     # all values are per second
+    high = FloatField(null=False)
+    critical = FloatField(null=False)
+
+
+class TemperaturesPlugin(Plugin):
+    models = [Temperatures]
+
+    def execute(self):
+        for sensor, temps in psutil.sensors_temperatures(config.TEMPERATURE_USE_FAHRENHEIT).items():
+            for temp in temps:
+                entry = Temperatures()
+                entry.sensor = sensor
+                entry.sensor_label = temp.label
+                entry.current = temp.current
+                entry.high = temp.high
+                entry.critical = temp.critical
+                entry.save()