Implemented collector utility.

.gitignore

@@ -0,0 +1,145 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+
+.vscode
+*.code-workspace
+
+# Local History for Visual Studio Code
+.history/

collector.py

@@ -0,0 +1,62 @@
+import config
+import database
+import signal
+from threading import Event
+from plugins.cpu_plugin import CpuPlugin
+from plugins.memory_plugin import MemoryPlugin
+from plugins.disk_plugin import DiskPlugin
+from plugins.network_plugin import NetworkPlugin
+from plugins.temperatures_plugin import TemperaturesPlugin
+from plugins.ping_plugin import PingPlugin
+
+class Collector(object):
+
+    def __init__(self):
+        self.plugins = [
+            CpuPlugin(),
+            MemoryPlugin(),
+            DiskPlugin(),
+            NetworkPlugin(),
+            TemperaturesPlugin(),
+            PingPlugin()
+        ]
+        self.event = Event()
+
+    def collect_models(self):
+        models = []
+        for plugin in self.plugins:
+            models.extend(plugin.models)
+        return models
+
+    def initialize(self):
+        models = self.collect_models()
+        database.initialize_db()
+        database.DB.create_tables(models)
+
+        signal.signal(signal.SIGHUP, self.abort)
+        signal.signal(signal.SIGTERM, self.abort)
+        signal.signal(signal.SIGINT, self.abort)
+
+    def run(self):
+        self.initialize()
+        print(f'Started.')
+
+        while not self.event.is_set():
+            for plugin in self.plugins:
+                # try:
+                plugin.execute()
+                # except BaseException as ex:
+                    # print(ex)
+            
+            self.event.wait(config.INTERVAL)
+            # TODO: calculate wait time based on execution time
+
+        print(f'Stopped.')
+
+    def abort(self, signum, frame):
+        print(f'Received signal {signum}, aborting...')
+        self.event.set()
+
+
+if __name__ == "__main__":
+    Collector().run()

config.py

@@ -0,0 +1,51 @@
+# Collect interval in seconds
+INTERVAL=30
+
+# Database URL, which defines connection settings.
+#
+# sqlite:///my_database.db 
+#   will create a SqliteDatabase instance for the file my_database.db in the current directory.
+#
+# sqlite:///:memory:
+#   will create an in-memory SqliteDatabase instance.
+#
+# postgresql://postgres:my_password@localhost:5432/my_database 
+#   will create a PostgresqlDatabase instance. A username and password are provided, as well as the host and port to connect to.
+#
+# mysql://user:passwd@ip:port/my_db 
+#   will create a MySQLDatabase instance for the local MySQL database my_db.
+#
+# mysql+pool://user:passwd@ip:port/my_db?max_connections=20&stale_timeout=300 
+#   will create a PooledMySQLDatabase instance for the local MySQL database my_db with max_connections set to 20 and a
+#   stale_timeout setting of 300 seconds.
+
+DATABASE_URL = 'sqlite:///data.db'
+
+
+# Plugin configuration
+
+
+### CPU
+# Store statistics per CPU, not only combined
+CPU_PER_CPU = True
+
+### Disk
+# How often to poll space information, as a number of INTERVALs
+DISK_SPACE_FREQUENCY = 10
+
+### Temperatures
+# If true, fahrenheit is used, otherwise celsius
+TEMPERATURE_USE_FAHRENHEIT = False
+
+### Ping
+# Ping hosts
+PING_HOSTS = [
+    '10.0.0.1',
+    '192.168.0.1',
+    '1.1.1.1',
+    'google.com',
+    'bing.com'
+]
+
+# How often to send pings, as a number of INTERVALs
+PING_FREQUENCY = 10

database.py

@@ -0,0 +1,13 @@
+from peewee import DatabaseProxy, Model
+from playhouse.db_url import connect
+import config
+
+DB = DatabaseProxy()
+
+class BaseModel(Model):
+    class Meta:
+        database = DB
+
+def initialize_db():
+    db = connect(config.DATABASE_URL)
+    DB.initialize(db)

plugins/cpu_plugin.py

@@ -0,0 +1,53 @@
+from datetime import datetime
+
+import psutil
+from peewee import *
+from playhouse.shortcuts import model_to_dict
+
+import config
+from database import BaseModel
+
+from .plugin import Plugin
+
+
+class Cpu(BaseModel):
+    time = DateTimeField(index=True, default=datetime.now)
+    cpu = SmallIntegerField(null=True)
+    idle_pct = FloatField(null=False)
+    user_pct = FloatField(null=False)
+    system_pct = FloatField(null=False)
+    nice_pct = FloatField(null=True)
+    iowait_pct = FloatField(null=True)
+    irq_pct = FloatField(null=True)
+    softirq_pct = FloatField(null=True)
+    freq_min = FloatField(null=True)
+    freq_current = FloatField(null=True)
+    freq_max = FloatField(null=True)
+
+
+class CpuPlugin(Plugin):
+    models = [Cpu]
+
+    def store(self, cpu, times, freq):
+        entry = Cpu()
+        entry.cpu = cpu
+        entry.idle_pct = times.idle
+        entry.user_pct = times.user
+        entry.system_pct = times.system
+        entry.nice_pct = getattr(times, 'nice', None)
+        entry.iowait_pct = getattr(times, 'iowait', None)
+        entry.irq_pct = getattr(times, 'irq', getattr(times, 'interrupt', None))
+        entry.softirq_pct = getattr(times, 'softirq', None)
+        entry.freq_min = getattr(freq, 'min', None)
+        entry.freq_current = getattr(freq, 'current', None)
+        entry.freq_max = getattr(freq, 'max', None)
+        entry.save()
+
+    def execute(self):
+        self.store(None, psutil.cpu_times_percent(percpu=False), psutil.cpu_freq(percpu=False))
+
+        if config.CPU_PER_CPU:
+            times = psutil.cpu_times_percent(percpu=True)
+            freqs = psutil.cpu_freq(percpu=True)
+            for i in range(len(times)):
+                self.store(i, times[i], freqs[i])

plugins/disk_plugin.py

@@ -0,0 +1,74 @@
+from collections import namedtuple
+from datetime import datetime
+
+import psutil
+from peewee import *
+from playhouse.shortcuts import model_to_dict
+
+import config
+from database import BaseModel
+
+from .plugin import Plugin
+
+
+class DiskUsage(BaseModel):
+    time = DateTimeField(index=True, default=datetime.now)
+    partition = TextField(null=False)
+    mountpoint = TextField(null=False)
+    total = BigIntegerField(null=False)
+    used = BigIntegerField(null=False)
+    free = BigIntegerField(null=False)
+
+
+class DiskIO(BaseModel):
+    time = DateTimeField(index=True, default=datetime.now)
+    disk = TextField(null=True)
+    read_count = FloatField(null=False)     # all values are per second
+    write_count = FloatField(null=False)
+    read_speed = FloatField(null=False)
+    write_speed = FloatField(null=False)
+
+
+class DiskPlugin(Plugin):
+    models = [DiskUsage, DiskIO]
+
+    def __init__(self):
+        self.__i = 0
+        self.__previous_io = {}
+
+    def store_io(self, disk, current):
+        previous = self.__previous_io.get(disk, current)
+
+        entry = DiskIO()
+        entry.disk = disk
+        entry.read_count = (current.read_count - previous.read_count) / config.INTERVAL
+        entry.write_count = (current.write_count - previous.write_count) / config.INTERVAL
+        entry.read_speed = (current.read_bytes - previous.read_bytes) / config.INTERVAL
+        entry.write_speed = (current.write_bytes - previous.write_bytes) / config.INTERVAL
+        entry.save()
+
+        self.__previous_io[disk] = current
+
+    def execute(self):
+
+        # Collect disk usage
+        if (self.__i % config.DISK_SPACE_FREQUENCY) == 0:
+            for partition in psutil.disk_partitions():
+                usage = psutil.disk_usage(partition.mountpoint)
+                
+                entry = DiskUsage()
+                entry.partition = partition.device
+                entry.mountpoint = partition.mountpoint
+                entry.total = usage.total
+                entry.used = usage.used
+                entry.free = usage.free
+                entry.save()
+
+        # Collect IO
+        self.store_io(None, psutil.disk_io_counters(perdisk=False))
+
+        io_reads = psutil.disk_io_counters(perdisk=True)
+        for disk, current in io_reads.items():
+            self.store_io(disk, current)
+
+        self.__i += 1

plugins/memory_plugin.py

@@ -0,0 +1,46 @@
+from datetime import datetime
+
+import psutil
+from peewee import *
+
+import config
+from database import BaseModel
+
+from .plugin import Plugin
+
+
+class Memory(BaseModel):
+    time = DateTimeField(index=True, default=datetime.now)
+    total = BigIntegerField(null=False)
+    available = BigIntegerField(null=False)
+    used = BigIntegerField(null=False)
+    free = BigIntegerField(null=False)
+    active = BigIntegerField(null=True)
+    inactive = BigIntegerField(null=True)
+    buffers = BigIntegerField(null=True)
+    cached = BigIntegerField(null=True)
+    swap_total = BigIntegerField(null=False)
+    swap_used = BigIntegerField(null=False)
+    swap_free = BigIntegerField(null=False)
+
+
+class MemoryPlugin(Plugin):
+    models = [Memory]
+
+    def execute(self):
+        vmem = psutil.virtual_memory()
+        swap = psutil.swap_memory()
+
+        entry = Memory()
+        entry.total = vmem.total
+        entry.available = vmem.available
+        entry.used = vmem.used
+        entry.free = vmem.free
+        entry.active = getattr(vmem, 'active', None)
+        entry.inactive = getattr(vmem, 'inactive', None)
+        entry.buffers = getattr(vmem, 'buffers', None)
+        entry.cached = getattr(vmem, 'cached', None)
+        entry.swap_total = swap.total
+        entry.swap_free = swap.free
+        entry.swap_used = swap.used
+        entry.save()

plugins/network_plugin.py

@@ -0,0 +1,48 @@
+from collections import namedtuple
+from datetime import datetime
+
+import psutil
+from peewee import *
+from playhouse.shortcuts import model_to_dict
+
+import config
+from database import BaseModel
+
+from .plugin import Plugin
+
+
+class NetworkIO(BaseModel):
+    time = DateTimeField(index=True, default=datetime.now)
+    nic = TextField(null=True)
+    packets_sent = FloatField(null=False)     # all values are per second
+    packets_recv = FloatField(null=False)
+    bytes_sent = FloatField(null=False)
+    bytes_recv = FloatField(null=False)
+
+
+class NetworkPlugin(Plugin):
+    models = [NetworkIO]
+
+    def __init__(self):
+        self.__previous_io = {}
+
+    def store_io(self, nic, current):
+        previous = self.__previous_io.get(nic, current)
+
+        entry = NetworkIO()
+        entry.nic = nic
+        entry.packets_sent = (current.packets_sent - previous.packets_sent) / config.INTERVAL
+        entry.packets_recv = (current.packets_recv - previous.packets_recv) / config.INTERVAL
+        entry.bytes_sent = (current.bytes_sent - previous.bytes_sent) / config.INTERVAL
+        entry.bytes_recv = (current.bytes_recv - previous.bytes_recv) / config.INTERVAL
+        entry.save()
+
+        self.__previous_io[nic] = current
+
+    def execute(self):
+
+        self.store_io(None, psutil.net_io_counters(pernic=False))
+
+        io_reads = psutil.net_io_counters(pernic=True)
+        for nic, current in io_reads.items():
+            self.store_io(nic, current)

plugins/ping_plugin.py

@@ -0,0 +1,61 @@
+import asyncio
+import re
+import subprocess
+from datetime import datetime
+
+import psutil
+from peewee import *
+from playhouse.shortcuts import model_to_dict
+
+import config
+from database import BaseModel
+
+from .plugin import Plugin
+
+
+class Ping(BaseModel):
+    time = DateTimeField(index=True, default=datetime.now)
+    host = TextField(null=False)
+    ping = FloatField(null=True) # null = timeout or error
+
+
+class PingPlugin(Plugin):
+    models = [Ping]
+
+    def __init__(self):
+        self.__timeout = config.INTERVAL // 3
+        self.__i = 0
+
+    async def do_ping(self, host):
+        command = ['ping', '-c', '1', '-W', str(self.__timeout), host]
+        proc = await asyncio.create_subprocess_shell(' '.join(command), 
+                                                     stdout=asyncio.subprocess.PIPE)
+
+        stdout,_ = await proc.communicate()
+        stdout = stdout.decode()
+
+        entry = Ping()
+        entry.host = host
+        entry.ping = None
+
+        match = re.search(r'time=([\d\.]+) ms', stdout)
+        if match is not None:
+            entry.ping = float(match.group(1))
+
+        entry.save()
+        print(model_to_dict(entry))
+
+    async def execute_internal(self):
+        await asyncio.gather(*[self.do_ping(host) for host in config.PING_HOSTS])
+
+    def execute(self):
+        if (self.__i % config.PING_FREQUENCY) == 0:
+            if getattr(asyncio, 'run', None) is not None:
+                # Python 3.7+
+                asyncio.run(self.execute_internal())
+            else:
+                loop = asyncio.get_event_loop()
+                loop.run_until_complete(self.execute_internal())
+                loop.close()
+        
+        self.__i += 1

plugins/plugin.py

@@ -0,0 +1,10 @@
+from abc import ABC, abstractmethod
+from typing import Tuple
+
+
+class Plugin(ABC):
+    models = []
+
+    @abstractmethod
+    def execute(self) -> None:
+        pass

plugins/temperatures_plugin.py

@@ -0,0 +1,35 @@
+from collections import namedtuple
+from datetime import datetime
+
+import psutil
+from peewee import *
+from playhouse.shortcuts import model_to_dict
+
+import config
+from database import BaseModel
+
+from .plugin import Plugin
+
+
+class Temperatures(BaseModel):
+    time = DateTimeField(index=True, default=datetime.now)
+    sensor = TextField(null=False)
+    sensor_label = TextField(null=False)
+    current = FloatField(null=False)     # all values are per second
+    high = FloatField(null=False)
+    critical = FloatField(null=False)
+
+
+class TemperaturesPlugin(Plugin):
+    models = [Temperatures]
+
+    def execute(self):
+        for sensor, temps in psutil.sensors_temperatures(config.TEMPERATURE_USE_FAHRENHEIT).items():
+            for temp in temps:
+                entry = Temperatures()
+                entry.sensor = sensor
+                entry.sensor_label = temp.label
+                entry.current = temp.current
+                entry.high = temp.high
+                entry.critical = temp.critical
+                entry.save()

requirements.txt

@@ -0,0 +1 @@
+peewee