From bcaf5144fb539278a6819aedfcd3ca90c9a409ce Mon Sep 17 00:00:00 2001 From: ducoterra Date: Sun, 5 Apr 2020 14:15:31 -0400 Subject: [PATCH] init --- .gitignore | 3 ++ collect.py | 3 ++ collector.json | 27 ++++++++++++++++ collector/__init__.py | 71 ++++++++++++++++++++++++++++++++++++++++++ collector/collector.py | 41 ++++++++++++++++++++++++ collector/helper.py | 14 +++++++++ collector/sender.py | 14 +++++++++ 7 files changed, 173 insertions(+) create mode 100644 .gitignore create mode 100755 collect.py create mode 100644 collector.json create mode 100644 collector/__init__.py create mode 100644 collector/collector.py create mode 100644 collector/helper.py create mode 100644 collector/sender.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..739eeb2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.vscode/ +venv/ +__pycache__/ \ No newline at end of file diff --git a/collect.py b/collect.py new file mode 100755 index 0000000..1c28ec5 --- /dev/null +++ b/collect.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python3 + +from collector import * \ No newline at end of file diff --git a/collector.json b/collector.json new file mode 100644 index 0000000..c474222 --- /dev/null +++ b/collector.json @@ -0,0 +1,27 @@ +{ + "graphite": { + "server": "graphite.ducoterra.net", + "port": 2003 + }, + "os": "osx", + "data": { + "storage": [ + { + "path": "/Users/ducoterra", + "name": "home" + }, + { + "path": ".", + "name": "curdir" + }, + { + "path": "..", + "name": "outdir" + }, + { + "path": "/Users", + "name": "users" + } + ] + } +} \ No newline at end of file diff --git a/collector/__init__.py b/collector/__init__.py new file mode 100644 index 0000000..9820ec4 --- /dev/null +++ b/collector/__init__.py @@ -0,0 +1,71 @@ +import os +import sys +import json +import functools +from collector.sender import send +from collector.collector import * +from collector.helper import * + +CONFIG_FILE = os.getenv("COLLECTOR_CONF", "collector.json") +REQUIRED = ["graphite","os","data"] +VALID_DATAPOINTS = ["storage"] +VALID_OS = ["linux", "osx"] +REQUIRED_STORAGE = ["path", "name"] +CONFIG = {} +HOSTNAME = get_hostname() + +with open(CONFIG_FILE) as f: + CONFIG = json.loads(f.read()) + +# Check top level +missing = get_missing(REQUIRED, CONFIG.keys()) +if len(missing) > 0: + print(f"Missing: {', '.join(missing)}") + sys.exit(1) + +# Now that we know we have a graphite server +sender = functools.partial( + send, + server = CONFIG["graphite"]["server"], + port = CONFIG["graphite"]["port"] +) + +# Check data exists +data = CONFIG["data"] +if type(data) != dict: + print("Config at 'data' missing values.") + sys.exit(1) +data_points = data.keys() +if not len(data_points) > 0: + print("No data points found in config.") + sys.exit(1) +unknown = get_missing(data_points, VALID_DATAPOINTS) +if len(unknown) > 0: + print(f"Unknown data: {', '.join(unknown)}") + sys.exit(1) + +# Check storage +if "storage" in data_points: + storage_data = data["storage"] + # Check if storage is a list + if type(storage_data) != list: + print("storage data must be a list") + sys.exit(1) + # Check if storage is not empty + if not len(storage_data) > 0: + print("Storage data cannot be empty") + sys.exit(1) + for item in storage_data: + # Check if the storage item is a dict + if type(item) != dict: + print(f"Storage data at {item} is not an object.") + # Make sure each storage item has the required fields + missing = get_missing(REQUIRED_STORAGE, item.keys()) + if len(missing) > 0: + print(f"Storage data at {item} missing: {', '.join(missing)}") + # Collect data + else: + name = item["name"] + path = item["path"] + storage_sender = functools.partial(sender, f"{HOSTNAME}.storage.{name}") + thread(collect_storage, storage_sender, path) \ No newline at end of file diff --git a/collector/collector.py b/collector/collector.py new file mode 100644 index 0000000..6839c87 --- /dev/null +++ b/collector/collector.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +import threading +import subprocess +import re +import functools +import random +import time + +from .sender import send + +# Disk Space +# CPU +# MEM +# Temperatures +# Network in/out +# Backups (Plus 5 day forecast) + +def handle_stdout(stdout): + return stdout.decode("utf-8").strip() + +def nice_run(*args, **kwargs): + run = subprocess.run(*args, capture_output=True, **kwargs) + return handle_stdout(run.stdout) + +def get_hostname(): + return nice_run(["hostname"]) + +def clean_storage(storage): + group = re.search(r"([0-9]*.?[0-9]+[\w]+)", storage) + return group.group(1) + +def collect_temp(): + pass + +def collect_compute(): + pass + +def collect_storage(path): + storage = nice_run(["du", "-sk", path]) + return clean_storage(storage) \ No newline at end of file diff --git a/collector/helper.py b/collector/helper.py new file mode 100644 index 0000000..3090367 --- /dev/null +++ b/collector/helper.py @@ -0,0 +1,14 @@ +import functools +import threading + +def thread(func, after, *args, **kwargs): + func = functools.partial(func, *args, **kwargs) + call = lambda after, func: after(func()) + t = threading.Thread( + target = call, + args = (after, func) + ) + t.start() + +def get_missing(required, provided): + return list(filter(lambda item: item not in provided, required)) \ No newline at end of file diff --git a/collector/sender.py b/collector/sender.py new file mode 100644 index 0000000..7ce90dc --- /dev/null +++ b/collector/sender.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + +import socket +import time + +def send(metric, data, server = None, port = None): + received = int(time.time()) + + with socket.socket() as sock: + msg = f"{metric} {data} {received}\n" + # msg = f"mainframe.temp 40 {received}\n" + msg = msg.encode("utf-8") + sock.connect((server, port)) + sock.sendall(msg) \ No newline at end of file