Added stocks plugin

collector.py

@@ -1,30 +1,38 @@
 import config
 import database
 import signal
+from apscheduler.schedulers.blocking import BlockingScheduler
 from threading import Event
 from plugins.system.cpu_plugin import CpuPlugin
 from plugins.system.memory_plugin import MemoryPlugin
-from plugins.system.disk_plugin import DiskPlugin
+from plugins.system.disk_plugin import DiskIOPlugin, DiskUsagePlugin
 from plugins.system.network_plugin import NetworkPlugin
 from plugins.system.temperatures_plugin import TemperaturesPlugin
 from plugins.system.ping_plugin import PingPlugin
+from plugins.finance.stocks_plugin import StocksPlugin
 from plugins.finance.robor_plugin import RoborPlugin
+import logging
+
 
 class Collector(object):
 
     def __init__(self):
         self.plugins = [
+            # system
             CpuPlugin(),
             MemoryPlugin(),
-            DiskPlugin(),
+            DiskUsagePlugin(),
+            DiskIOPlugin(),
             NetworkPlugin(),
             TemperaturesPlugin(),
             PingPlugin(),
 
             # finance
+            StocksPlugin(),
             RoborPlugin()
         ]
         self.event = Event()
+        self.scheduler = BlockingScheduler()
 
     def collect_models(self):
         models = []
@@ -32,28 +40,28 @@ class Collector(object):
             models.extend(plugin.models)
         return models
 
-    def initialize(self):
+    def schedule_plugins(self):
+        for plugin in self.plugins:
+            self.scheduler.add_job(plugin.execute, 'interval', seconds=plugin.get_interval())
+
+    def run(self):
+        logging.basicConfig()
+        logging.getLogger('apscheduler').setLevel(logging.INFO)
+
         models = self.collect_models()
         database.initialize_db()
         database.DB.create_tables(models)
 
-        signal.signal(signal.SIGHUP, self.abort)
+        self.schedule_plugins()
-        signal.signal(signal.SIGTERM, self.abort)
+        # signal.signal(signal.SIGHUP, self.abort)
-        signal.signal(signal.SIGINT, self.abort)
+        # signal.signal(signal.SIGTERM, self.abort)
-
+        # signal.signal(signal.SIGINT, self.abort)
-    def run(self):
-        self.initialize()
         print(f'Started.')
 
-        while not self.event.is_set():
+        try:
-            for plugin in self.plugins:
+            self.scheduler.start()
-                # try:
+        except (KeyboardInterrupt, SystemExit):
-                plugin.execute()
+            pass
-                # except BaseException as ex:
-                    # print(ex)
-            
-            self.event.wait(config.INTERVAL)
-            # TODO: calculate wait time based on execution time
 
         print(f'Stopped.')
 

config.py

@@ -1,5 +1,5 @@
 # Collect interval in seconds
-INTERVAL=30
+DEFAULT_INTERVAL = 30
 
 # Database URL, which defines connection settings.
 #
@@ -26,19 +26,30 @@ DATABASE_URL = 'sqlite:///data.db'
 
 
 ### CPU
+CPU_INTERVAL = DEFAULT_INTERVAL
 # Store statistics per CPU, not only combined
 CPU_PER_CPU = True
 
 ### Disk
-# How often to poll space information, as a number of INTERVALs
+DISK_USAGE_INTERVAL = DEFAULT_INTERVAL * 10
-DISK_SPACE_FREQUENCY = 10
+DISK_IO_INTERVAL = DEFAULT_INTERVAL
+
+### Memory
+MEMORY_INTERVAL = DEFAULT_INTERVAL
+
+### Network
+NETWORK_INTERVAL = DEFAULT_INTERVAL
 
 ### Temperatures
+TEMPERATURE_INTERVAL = DEFAULT_INTERVAL
+
 # If true, fahrenheit is used, otherwise celsius
 TEMPERATURE_USE_FAHRENHEIT = False
 
 ### Ping
 # Ping hosts
+PING_INTERVAL = 5 * 60              # every 5 min
+
 PING_HOSTS = [
     '10.0.0.1',
     '192.168.0.1',
@@ -47,11 +58,23 @@ PING_HOSTS = [
     'bing.com'
 ]
 
-# How often to send pings, as a number of INTERVALs
+### Stocks
-PING_FREQUENCY = 10
+STOCKS_INTERVAL = 12 * 60 * 60      # updates daily
+
+STOCKS_TICKERS = {
+    'MCRO.L' : 'Micro Focus International PLC',
+    '^GSPC' : 'S&P 500',
+    '^SPEUP' : 'S&P 350 Europe',
+    '^SPG1200' : 'S&P 1200 Global',
+    '^IXIC' : 'NASDAQ Composite',
+    'BTC-USD' : 'Bitcoin USD',
+    'ETH-USD' : 'Ethereum USD',
+}
 
 ### ROBOR
 # Romanian Interbank Offer Rate
+ROBOR_INTERVAL = 12 * 60 * 60       # updates daily, every 12 hours should be fine
+
 ROBOR_FIELDS = [
     'ROBOR 6M'
 ]

plugins/finance/robor_plugin.py

@@ -22,6 +22,9 @@ class RoborPlugin(Plugin):
     def __init__(self):
         self.__table = None
 
+    def get_interval(self):
+        return config.ROBOR_INTERVAL
+
     def get_column_index(self, table, column_name):
         header_row = table.find('tr')
         for elem in header_row.iter('th'):
@@ -74,7 +77,6 @@ class RoborPlugin(Plugin):
                 entry.field = field
                 entry.value = value
                 entry.save()
-                print(model_to_dict(entry))
 
 
 

plugins/finance/stocks_plugin.py

@@ -0,0 +1,56 @@
+import datetime
+
+from peewee import *
+from playhouse.shortcuts import model_to_dict
+
+import config
+from database import BaseModel
+from plugins.plugin import Plugin
+import yfinance as yf
+
+
+class Stocks(BaseModel):
+    date = DateTimeField(index=True, default=datetime.datetime.now(), null=False)
+    ticker = TextField(null=False)
+    label = TextField(null=False)
+    value_open = FloatField(null=False)
+    value_close = FloatField(null=False)
+    value_high = FloatField(null=False)
+    value_low = FloatField(null=False)
+
+
+class StocksPlugin(Plugin):
+    models = [Stocks]
+
+    def get_interval(self):
+        return config.STOCKS_INTERVAL
+
+    def execute(self):
+        for ticker, label in config.STOCKS_TICKERS.items():
+            # Get last existing date
+            latest_date = Stocks.select(Stocks.date) \
+                            .order_by(Stocks.date.desc()) \
+                            .limit(1) \
+                            .scalar()
+
+            try:
+                yfticker = yf.Ticker(ticker)
+
+                if latest_date is None:
+                    data = yfticker.history(period='max')
+                else:
+                    data = yfticker.history(start=latest_date + datetime.timedelta(seconds=1))
+
+                for row in data.itertuples():
+                    entry = Stocks()
+                    entry.date = row.Index.to_pydatetime()
+                    entry.ticker = ticker
+                    entry.label = label
+                    entry.value_open = row.Open
+                    entry.value_close = row.Close
+                    entry.value_high = row.High
+                    entry.value_low = row.Low
+                    entry.save()
+                    print(model_to_dict(entry))
+            except BaseException as e:
+                print(e)

plugins/plugin.py

@@ -5,6 +5,10 @@ from typing import Tuple
 class Plugin(ABC):
     models = []
 
+    @abstractmethod
+    def get_interval(self) -> int:
+        pass
+
     @abstractmethod
     def execute(self) -> None:
         pass

plugins/system/cpu_plugin.py

@@ -27,6 +27,9 @@ class Cpu(BaseModel):
 class CpuPlugin(Plugin):
     models = [Cpu]
 
+    def get_interval(self):
+        return config.CPU_INTERVAL
+
     def store(self, cpu, times, freq):
         entry = Cpu()
         entry.cpu = cpu

plugins/system/disk_plugin.py

@@ -28,46 +28,52 @@ class DiskIO(BaseModel):
     write_speed = FloatField(null=False)
 
 
-class DiskPlugin(Plugin):
+class DiskUsagePlugin(Plugin):
-    models = [DiskUsage, DiskIO]
+    models = [DiskUsage]
+
+    def get_interval(self):
+        return config.DISK_USAGE_INTERVAL
+
+    def execute(self):
+        for partition in psutil.disk_partitions():
+            usage = psutil.disk_usage(partition.mountpoint)
+            
+            entry = DiskUsage()
+            entry.partition = partition.device
+            entry.mountpoint = partition.mountpoint
+            entry.total = usage.total
+            entry.used = usage.used
+            entry.free = usage.free
+            entry.save()
+
+
+class DiskIOPlugin(Plugin):
+    models = [DiskIO]
 
     def __init__(self):
-        self.__i = 0
         self.__previous_io = {}
 
+    def get_interval(self):
+        return config.DISK_IO_INTERVAL
+
     def store_io(self, disk, current):
         previous = self.__previous_io.get(disk, current)
 
         entry = DiskIO()
         entry.disk = disk
-        entry.read_count = (current.read_count - previous.read_count) / config.INTERVAL
+        entry.read_count = (current.read_count - previous.read_count) / self.get_interval()
-        entry.write_count = (current.write_count - previous.write_count) / config.INTERVAL
+        entry.write_count = (current.write_count - previous.write_count) / self.get_interval()
-        entry.read_speed = (current.read_bytes - previous.read_bytes) / config.INTERVAL
+        entry.read_speed = (current.read_bytes - previous.read_bytes) / self.get_interval()
-        entry.write_speed = (current.write_bytes - previous.write_bytes) / config.INTERVAL
+        entry.write_speed = (current.write_bytes - previous.write_bytes) / self.get_interval()
         entry.save()
 
         self.__previous_io[disk] = current
 
     def execute(self):
 
-        # Collect disk usage
-        if (self.__i % config.DISK_SPACE_FREQUENCY) == 0:
-            for partition in psutil.disk_partitions():
-                usage = psutil.disk_usage(partition.mountpoint)
-                
-                entry = DiskUsage()
-                entry.partition = partition.device
-                entry.mountpoint = partition.mountpoint
-                entry.total = usage.total
-                entry.used = usage.used
-                entry.free = usage.free
-                entry.save()
-
-        # Collect IO
         self.store_io(None, psutil.disk_io_counters(perdisk=False))
 
         io_reads = psutil.disk_io_counters(perdisk=True)
         for disk, current in io_reads.items():
             self.store_io(disk, current)
 
-        self.__i += 1

plugins/system/memory_plugin.py

@@ -26,6 +26,9 @@ class Memory(BaseModel):
 class MemoryPlugin(Plugin):
     models = [Memory]
 
+    def get_interval(self):
+        return config.MEMORY_INTERVAL
+
     def execute(self):
         vmem = psutil.virtual_memory()
         swap = psutil.swap_memory()

plugins/system/network_plugin.py

@@ -25,15 +25,18 @@ class NetworkPlugin(Plugin):
     def __init__(self):
         self.__previous_io = {}
 
+    def get_interval(self):
+        return config.NETWORK_INTERVAL
+
     def store_io(self, nic, current):
         previous = self.__previous_io.get(nic, current)
 
         entry = NetworkIO()
         entry.nic = nic
-        entry.packets_sent = (current.packets_sent - previous.packets_sent) / config.INTERVAL
+        entry.packets_sent = (current.packets_sent - previous.packets_sent) / self.get_interval()
-        entry.packets_recv = (current.packets_recv - previous.packets_recv) / config.INTERVAL
+        entry.packets_recv = (current.packets_recv - previous.packets_recv) / self.get_interval()
-        entry.bytes_sent = (current.bytes_sent - previous.bytes_sent) / config.INTERVAL
+        entry.bytes_sent = (current.bytes_sent - previous.bytes_sent) / self.get_interval()
-        entry.bytes_recv = (current.bytes_recv - previous.bytes_recv) / config.INTERVAL
+        entry.bytes_recv = (current.bytes_recv - previous.bytes_recv) / self.get_interval()
         entry.save()
 
         self.__previous_io[nic] = current

plugins/system/ping_plugin.py

@@ -22,8 +22,10 @@ class PingPlugin(Plugin):
     models = [Ping]
 
     def __init__(self):
-        self.__timeout = config.INTERVAL // 3
+        self.__timeout = config.PING_INTERVAL // 3
-        self.__i = 0
+
+    def get_interval(self):
+        return config.PING_INTERVAL
 
     async def do_ping(self, host):
         command = ['ping', '-c', '1', '-W', str(self.__timeout), host]
@@ -47,13 +49,12 @@ class PingPlugin(Plugin):
         await asyncio.gather(*[self.do_ping(host) for host in config.PING_HOSTS])
 
     def execute(self):
-        if (self.__i % config.PING_FREQUENCY) == 0:
+        if getattr(asyncio, 'run', None) is not None:
-            if getattr(asyncio, 'run', None) is not None:
+            # Python 3.7+
-                # Python 3.7+
+            asyncio.run(self.execute_internal())
-                asyncio.run(self.execute_internal())
+        else:
-            else:
+            loop = asyncio.get_event_loop()
-                loop = asyncio.get_event_loop()
+            loop.run_until_complete(self.execute_internal())
-                loop.run_until_complete(self.execute_internal())
+            loop.close()
-                loop.close()
+    
-        
+    
-        self.__i += 1

plugins/system/temperatures_plugin.py

@@ -22,6 +22,9 @@ class Temperatures(BaseModel):
 class TemperaturesPlugin(Plugin):
     models = [Temperatures]
 
+    def get_interval(self):
+        return config.TEMPERATURE_INTERVAL
+
     def execute(self):
         for sensor, temps in psutil.sensors_temperatures(config.TEMPERATURE_USE_FAHRENHEIT).items():
             for temp in temps:

requirements.txt

@@ -1,8 +1,12 @@
+apscheduler
 peewee
 
 # Used in most system plugins
 psutil
 
-# Finance plugins
+# robor plugin
 requests
-lxml
+lxml
+
+# stocks plugin
+yfinance