From daa8e5316c3af110018be6f69436b619adc606be Mon Sep 17 00:00:00 2001 From: Tiberiu Chibici Date: Fri, 17 Apr 2020 19:16:58 +0300 Subject: [PATCH] Added stocks plugin --- collector.py | 44 ++++++++++++--------- config.py | 33 +++++++++++++--- plugins/finance/robor_plugin.py | 4 +- plugins/finance/stocks_plugin.py | 56 +++++++++++++++++++++++++++ plugins/finance_plugin.py | 0 plugins/plugin.py | 4 ++ plugins/system/cpu_plugin.py | 3 ++ plugins/system/disk_plugin.py | 50 +++++++++++++----------- plugins/system/memory_plugin.py | 3 ++ plugins/system/network_plugin.py | 11 ++++-- plugins/system/ping_plugin.py | 25 ++++++------ plugins/system/temperatures_plugin.py | 3 ++ requirements.txt | 8 +++- 13 files changed, 180 insertions(+), 64 deletions(-) delete mode 100644 plugins/finance_plugin.py diff --git a/collector.py b/collector.py index deb3625..4f5eef6 100644 --- a/collector.py +++ b/collector.py @@ -1,30 +1,38 @@ import config import database import signal +from apscheduler.schedulers.blocking import BlockingScheduler from threading import Event from plugins.system.cpu_plugin import CpuPlugin from plugins.system.memory_plugin import MemoryPlugin -from plugins.system.disk_plugin import DiskPlugin +from plugins.system.disk_plugin import DiskIOPlugin, DiskUsagePlugin from plugins.system.network_plugin import NetworkPlugin from plugins.system.temperatures_plugin import TemperaturesPlugin from plugins.system.ping_plugin import PingPlugin +from plugins.finance.stocks_plugin import StocksPlugin from plugins.finance.robor_plugin import RoborPlugin +import logging + class Collector(object): def __init__(self): self.plugins = [ + # system CpuPlugin(), MemoryPlugin(), - DiskPlugin(), + DiskUsagePlugin(), + DiskIOPlugin(), NetworkPlugin(), TemperaturesPlugin(), PingPlugin(), # finance + StocksPlugin(), RoborPlugin() ] self.event = Event() + self.scheduler = BlockingScheduler() def collect_models(self): models = [] @@ -32,28 +40,28 @@ class Collector(object): models.extend(plugin.models) return models - def initialize(self): + def schedule_plugins(self): + for plugin in self.plugins: + self.scheduler.add_job(plugin.execute, 'interval', seconds=plugin.get_interval()) + + def run(self): + logging.basicConfig() + logging.getLogger('apscheduler').setLevel(logging.INFO) + models = self.collect_models() database.initialize_db() database.DB.create_tables(models) - signal.signal(signal.SIGHUP, self.abort) - signal.signal(signal.SIGTERM, self.abort) - signal.signal(signal.SIGINT, self.abort) - - def run(self): - self.initialize() + self.schedule_plugins() + # signal.signal(signal.SIGHUP, self.abort) + # signal.signal(signal.SIGTERM, self.abort) + # signal.signal(signal.SIGINT, self.abort) print(f'Started.') - while not self.event.is_set(): - for plugin in self.plugins: - # try: - plugin.execute() - # except BaseException as ex: - # print(ex) - - self.event.wait(config.INTERVAL) - # TODO: calculate wait time based on execution time + try: + self.scheduler.start() + except (KeyboardInterrupt, SystemExit): + pass print(f'Stopped.') diff --git a/config.py b/config.py index 942c4d2..6e107ed 100644 --- a/config.py +++ b/config.py @@ -1,5 +1,5 @@ # Collect interval in seconds -INTERVAL=30 +DEFAULT_INTERVAL = 30 # Database URL, which defines connection settings. # @@ -26,19 +26,30 @@ DATABASE_URL = 'sqlite:///data.db' ### CPU +CPU_INTERVAL = DEFAULT_INTERVAL # Store statistics per CPU, not only combined CPU_PER_CPU = True ### Disk -# How often to poll space information, as a number of INTERVALs -DISK_SPACE_FREQUENCY = 10 +DISK_USAGE_INTERVAL = DEFAULT_INTERVAL * 10 +DISK_IO_INTERVAL = DEFAULT_INTERVAL + +### Memory +MEMORY_INTERVAL = DEFAULT_INTERVAL + +### Network +NETWORK_INTERVAL = DEFAULT_INTERVAL ### Temperatures +TEMPERATURE_INTERVAL = DEFAULT_INTERVAL + # If true, fahrenheit is used, otherwise celsius TEMPERATURE_USE_FAHRENHEIT = False ### Ping # Ping hosts +PING_INTERVAL = 5 * 60 # every 5 min + PING_HOSTS = [ '10.0.0.1', '192.168.0.1', @@ -47,11 +58,23 @@ PING_HOSTS = [ 'bing.com' ] -# How often to send pings, as a number of INTERVALs -PING_FREQUENCY = 10 +### Stocks +STOCKS_INTERVAL = 12 * 60 * 60 # updates daily + +STOCKS_TICKERS = { + 'MCRO.L' : 'Micro Focus International PLC', + '^GSPC' : 'S&P 500', + '^SPEUP' : 'S&P 350 Europe', + '^SPG1200' : 'S&P 1200 Global', + '^IXIC' : 'NASDAQ Composite', + 'BTC-USD' : 'Bitcoin USD', + 'ETH-USD' : 'Ethereum USD', +} ### ROBOR # Romanian Interbank Offer Rate +ROBOR_INTERVAL = 12 * 60 * 60 # updates daily, every 12 hours should be fine + ROBOR_FIELDS = [ 'ROBOR 6M' ] diff --git a/plugins/finance/robor_plugin.py b/plugins/finance/robor_plugin.py index 75238a5..0441426 100644 --- a/plugins/finance/robor_plugin.py +++ b/plugins/finance/robor_plugin.py @@ -22,6 +22,9 @@ class RoborPlugin(Plugin): def __init__(self): self.__table = None + def get_interval(self): + return config.ROBOR_INTERVAL + def get_column_index(self, table, column_name): header_row = table.find('tr') for elem in header_row.iter('th'): @@ -74,7 +77,6 @@ class RoborPlugin(Plugin): entry.field = field entry.value = value entry.save() - print(model_to_dict(entry)) diff --git a/plugins/finance/stocks_plugin.py b/plugins/finance/stocks_plugin.py index e69de29..09c3242 100644 --- a/plugins/finance/stocks_plugin.py +++ b/plugins/finance/stocks_plugin.py @@ -0,0 +1,56 @@ +import datetime + +from peewee import * +from playhouse.shortcuts import model_to_dict + +import config +from database import BaseModel +from plugins.plugin import Plugin +import yfinance as yf + + +class Stocks(BaseModel): + date = DateTimeField(index=True, default=datetime.datetime.now(), null=False) + ticker = TextField(null=False) + label = TextField(null=False) + value_open = FloatField(null=False) + value_close = FloatField(null=False) + value_high = FloatField(null=False) + value_low = FloatField(null=False) + + +class StocksPlugin(Plugin): + models = [Stocks] + + def get_interval(self): + return config.STOCKS_INTERVAL + + def execute(self): + for ticker, label in config.STOCKS_TICKERS.items(): + # Get last existing date + latest_date = Stocks.select(Stocks.date) \ + .order_by(Stocks.date.desc()) \ + .limit(1) \ + .scalar() + + try: + yfticker = yf.Ticker(ticker) + + if latest_date is None: + data = yfticker.history(period='max') + else: + data = yfticker.history(start=latest_date + datetime.timedelta(seconds=1)) + + for row in data.itertuples(): + entry = Stocks() + entry.date = row.Index.to_pydatetime() + entry.ticker = ticker + entry.label = label + entry.value_open = row.Open + entry.value_close = row.Close + entry.value_high = row.High + entry.value_low = row.Low + entry.save() + print(model_to_dict(entry)) + except BaseException as e: + print(e) \ No newline at end of file diff --git a/plugins/finance_plugin.py b/plugins/finance_plugin.py deleted file mode 100644 index e69de29..0000000 diff --git a/plugins/plugin.py b/plugins/plugin.py index 465579b..61035c8 100644 --- a/plugins/plugin.py +++ b/plugins/plugin.py @@ -5,6 +5,10 @@ from typing import Tuple class Plugin(ABC): models = [] + @abstractmethod + def get_interval(self) -> int: + pass + @abstractmethod def execute(self) -> None: pass diff --git a/plugins/system/cpu_plugin.py b/plugins/system/cpu_plugin.py index e3c6e53..8616740 100644 --- a/plugins/system/cpu_plugin.py +++ b/plugins/system/cpu_plugin.py @@ -27,6 +27,9 @@ class Cpu(BaseModel): class CpuPlugin(Plugin): models = [Cpu] + def get_interval(self): + return config.CPU_INTERVAL + def store(self, cpu, times, freq): entry = Cpu() entry.cpu = cpu diff --git a/plugins/system/disk_plugin.py b/plugins/system/disk_plugin.py index 2894e38..398c435 100644 --- a/plugins/system/disk_plugin.py +++ b/plugins/system/disk_plugin.py @@ -28,46 +28,52 @@ class DiskIO(BaseModel): write_speed = FloatField(null=False) -class DiskPlugin(Plugin): - models = [DiskUsage, DiskIO] +class DiskUsagePlugin(Plugin): + models = [DiskUsage] + + def get_interval(self): + return config.DISK_USAGE_INTERVAL + + def execute(self): + for partition in psutil.disk_partitions(): + usage = psutil.disk_usage(partition.mountpoint) + + entry = DiskUsage() + entry.partition = partition.device + entry.mountpoint = partition.mountpoint + entry.total = usage.total + entry.used = usage.used + entry.free = usage.free + entry.save() + + +class DiskIOPlugin(Plugin): + models = [DiskIO] def __init__(self): - self.__i = 0 self.__previous_io = {} + def get_interval(self): + return config.DISK_IO_INTERVAL + def store_io(self, disk, current): previous = self.__previous_io.get(disk, current) entry = DiskIO() entry.disk = disk - entry.read_count = (current.read_count - previous.read_count) / config.INTERVAL - entry.write_count = (current.write_count - previous.write_count) / config.INTERVAL - entry.read_speed = (current.read_bytes - previous.read_bytes) / config.INTERVAL - entry.write_speed = (current.write_bytes - previous.write_bytes) / config.INTERVAL + entry.read_count = (current.read_count - previous.read_count) / self.get_interval() + entry.write_count = (current.write_count - previous.write_count) / self.get_interval() + entry.read_speed = (current.read_bytes - previous.read_bytes) / self.get_interval() + entry.write_speed = (current.write_bytes - previous.write_bytes) / self.get_interval() entry.save() self.__previous_io[disk] = current def execute(self): - # Collect disk usage - if (self.__i % config.DISK_SPACE_FREQUENCY) == 0: - for partition in psutil.disk_partitions(): - usage = psutil.disk_usage(partition.mountpoint) - - entry = DiskUsage() - entry.partition = partition.device - entry.mountpoint = partition.mountpoint - entry.total = usage.total - entry.used = usage.used - entry.free = usage.free - entry.save() - - # Collect IO self.store_io(None, psutil.disk_io_counters(perdisk=False)) io_reads = psutil.disk_io_counters(perdisk=True) for disk, current in io_reads.items(): self.store_io(disk, current) - self.__i += 1 diff --git a/plugins/system/memory_plugin.py b/plugins/system/memory_plugin.py index 7dd82fa..48b0b25 100644 --- a/plugins/system/memory_plugin.py +++ b/plugins/system/memory_plugin.py @@ -26,6 +26,9 @@ class Memory(BaseModel): class MemoryPlugin(Plugin): models = [Memory] + def get_interval(self): + return config.MEMORY_INTERVAL + def execute(self): vmem = psutil.virtual_memory() swap = psutil.swap_memory() diff --git a/plugins/system/network_plugin.py b/plugins/system/network_plugin.py index 7791f1e..ab3ba2d 100644 --- a/plugins/system/network_plugin.py +++ b/plugins/system/network_plugin.py @@ -25,15 +25,18 @@ class NetworkPlugin(Plugin): def __init__(self): self.__previous_io = {} + def get_interval(self): + return config.NETWORK_INTERVAL + def store_io(self, nic, current): previous = self.__previous_io.get(nic, current) entry = NetworkIO() entry.nic = nic - entry.packets_sent = (current.packets_sent - previous.packets_sent) / config.INTERVAL - entry.packets_recv = (current.packets_recv - previous.packets_recv) / config.INTERVAL - entry.bytes_sent = (current.bytes_sent - previous.bytes_sent) / config.INTERVAL - entry.bytes_recv = (current.bytes_recv - previous.bytes_recv) / config.INTERVAL + entry.packets_sent = (current.packets_sent - previous.packets_sent) / self.get_interval() + entry.packets_recv = (current.packets_recv - previous.packets_recv) / self.get_interval() + entry.bytes_sent = (current.bytes_sent - previous.bytes_sent) / self.get_interval() + entry.bytes_recv = (current.bytes_recv - previous.bytes_recv) / self.get_interval() entry.save() self.__previous_io[nic] = current diff --git a/plugins/system/ping_plugin.py b/plugins/system/ping_plugin.py index 0a85108..8d1ea0c 100644 --- a/plugins/system/ping_plugin.py +++ b/plugins/system/ping_plugin.py @@ -22,8 +22,10 @@ class PingPlugin(Plugin): models = [Ping] def __init__(self): - self.__timeout = config.INTERVAL // 3 - self.__i = 0 + self.__timeout = config.PING_INTERVAL // 3 + + def get_interval(self): + return config.PING_INTERVAL async def do_ping(self, host): command = ['ping', '-c', '1', '-W', str(self.__timeout), host] @@ -47,13 +49,12 @@ class PingPlugin(Plugin): await asyncio.gather(*[self.do_ping(host) for host in config.PING_HOSTS]) def execute(self): - if (self.__i % config.PING_FREQUENCY) == 0: - if getattr(asyncio, 'run', None) is not None: - # Python 3.7+ - asyncio.run(self.execute_internal()) - else: - loop = asyncio.get_event_loop() - loop.run_until_complete(self.execute_internal()) - loop.close() - - self.__i += 1 + if getattr(asyncio, 'run', None) is not None: + # Python 3.7+ + asyncio.run(self.execute_internal()) + else: + loop = asyncio.get_event_loop() + loop.run_until_complete(self.execute_internal()) + loop.close() + + diff --git a/plugins/system/temperatures_plugin.py b/plugins/system/temperatures_plugin.py index 4d4d6fb..0060380 100644 --- a/plugins/system/temperatures_plugin.py +++ b/plugins/system/temperatures_plugin.py @@ -22,6 +22,9 @@ class Temperatures(BaseModel): class TemperaturesPlugin(Plugin): models = [Temperatures] + def get_interval(self): + return config.TEMPERATURE_INTERVAL + def execute(self): for sensor, temps in psutil.sensors_temperatures(config.TEMPERATURE_USE_FAHRENHEIT).items(): for temp in temps: diff --git a/requirements.txt b/requirements.txt index 2e573ca..7f4161a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,12 @@ +apscheduler peewee # Used in most system plugins psutil -# Finance plugins +# robor plugin requests -lxml \ No newline at end of file +lxml + +# stocks plugin +yfinance \ No newline at end of file