Source code for promgen.models

# Copyright (c) 2017 LINE Corporation
# These sources are released under the terms of the MIT license: see LICENSE

import json
import logging

import django.contrib.sites.models
from django.conf import settings
from django.contrib.contenttypes.fields import (GenericForeignKey,
                                                GenericRelation)
from django.contrib.contenttypes.models import ContentType
from django.db import models, transaction
from django.forms.models import model_to_dict
from django.urls import reverse
from django.utils import timezone
from django.utils.functional import cached_property
from django.utils.text import slugify

import promgen.templatetags.promgen as macro
from promgen import plugins, tests, validators
from promgen.shortcuts import resolve_domain

logger = logging.getLogger(__name__)


[docs]class Site(django.contrib.sites.models.Site): # Proxy model for sites so that we can easily # query our related Rules rule_set = GenericRelation('promgen.Rule', for_concrete_model=False) def get_absolute_url(self): return reverse("site-detail") class Meta: proxy = True
class ObjectFilterManager(models.Manager): def create(self, *args, **kwargs): if 'obj' in kwargs: obj = kwargs.pop('obj') kwargs['object_id'] = obj.id kwargs['content_type_id'] = ContentType.objects.get_for_model(obj).id return self.get_queryset().create(*args, **kwargs) def filter(self, *args, **kwargs): if 'obj' in kwargs: obj = kwargs.pop('obj') kwargs['object_id'] = obj.id kwargs['content_type_id'] = ContentType.objects.get_for_model(obj).id return self.get_queryset().filter(*args, **kwargs) def get_or_create(self, *args, **kwargs): if "obj" in kwargs: obj = kwargs.pop("obj") kwargs["object_id"] = obj.id kwargs["content_type_id"] = ContentType.objects.get_for_model(obj).id if "defaults" in kwargs and "obj" in kwargs["defaults"]: obj = kwargs["defaults"].pop("obj") kwargs["defaults"]["object_id"] = obj.id kwargs["defaults"]["content_type_id"] = ContentType.objects.get_for_model( obj ).id return self.get_queryset().get_or_create(*args, **kwargs)
[docs]class Sender(models.Model): objects = ObjectFilterManager() sender = models.CharField(max_length=128) value = models.CharField(max_length=128) alias = models.CharField(max_length=128, blank=True) content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, limit_choices_to=( models.Q(app_label='auth', model='user') | models.Q(app_label='promgen', model='project') | models.Q(app_label='promgen', model='service')) ) object_id = models.PositiveIntegerField() content_object = GenericForeignKey('content_type', 'object_id') owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True) enabled = models.BooleanField(default=True) def show_value(self): if self.alias: return self.alias return self.value show_value.short_description = 'Value' def __str__(self): return '{}:{}'.format(self.sender, self.show_value())
[docs] @classmethod def driver_set(cls): '''Return the list of drivers for Sender model''' for entry in plugins.notifications(): try: yield entry.module_name, entry.load() except ImportError: logger.warning('Error importing %s', entry.module_name)
__driver = {} @property def driver(self): '''Return configured driver for Sender model instance''' if self.sender in self.__driver: return self.__driver[self.sender] for entry in plugins.notifications(): try: self.__driver[entry.module_name] = entry.load()() except ImportError: logger.warning('Error importing %s', entry.module_name) return self.__driver[self.sender]
[docs] def test(self): ''' Test sender plugin Uses the same test json from our unittests but subs in the currently tested object as part of the test data ''' data = tests.Data("examples", "alertmanager.json").json() if hasattr(self.content_object, 'name'): data['commonLabels'][self.content_type.name] = self.content_object.name for alert in data.get('alerts', []): alert['labels'][self.content_type.name] = self.content_object.name from promgen import tasks tasks.send_alert(self.sender, self.value, data)
[docs] def filtered(self, alert): """ Check filters for a specific sender If no filters are defined, then we let the message through If filters are defined, then we check to see if at least one filter matches If no filters match, then we assume it's filtered out """ logger.debug("Checking labels %s", alert["commonLabels"]) # If we have no specific whitelist, then we let everything through if self.filter_set.count() == 0: return False # If we have filters defined, then we need to check to see if our # filters match for f in self.filter_set.all(): logger.debug("Checking filter %s %s", f.name, f.value) if alert["commonLabels"].get(f.name) == f.value: return False # If none of our filters match, then we blacklist this sender return True
[docs]class Filter(models.Model): sender = models.ForeignKey("Sender", on_delete=models.CASCADE) name = models.CharField(max_length=128) value = models.CharField(max_length=128) class Meta: ordering = ("sender", "name", "value") unique_together = (("sender", "name", "value"),)
[docs]class Shard(models.Model): name = models.CharField(max_length=128, unique=True, validators=[validators.labelvalue]) url = models.URLField(max_length=256) proxy = models.BooleanField(default=False, help_text='Queries can be proxied to these shards') enabled = models.BooleanField(default=True, help_text='Able to register new Services and Projects') class Meta: ordering = ['name'] def get_absolute_url(self): return reverse('datasource-detail', kwargs={'pk': self.pk}) def __str__(self): if self.enabled: return self.name return self.name + ' (disabled)'
[docs]class Service(models.Model): name = models.CharField(max_length=128, unique=True, validators=[validators.labelvalue]) description = models.TextField(blank=True) owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True, default=None) notifiers = GenericRelation(Sender) rule_set = GenericRelation('Rule') class Meta: ordering = ['name'] def get_absolute_url(self): return reverse('service-detail', kwargs={'pk': self.pk}) def __str__(self): return self.name @classmethod def default(cls, service_name='Default', shard_name='Default'): shard, created = Shard.objects.get_or_create( name=shard_name ) if created: logger.info('Created default shard') service, created = cls.objects.get_or_create( name=service_name, defaults={'shard': shard} ) if created: logger.info('Created default service') return service
[docs]class Project(models.Model): name = models.CharField(max_length=128, unique=True, validators=[validators.labelvalue]) description = models.TextField(blank=True) owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True, default=None) service = models.ForeignKey('promgen.Service', on_delete=models.CASCADE) shard = models.ForeignKey('promgen.Shard', on_delete=models.CASCADE) farm = models.ForeignKey('promgen.Farm', blank=True, null=True, on_delete=models.SET_NULL) notifiers = GenericRelation(Sender) rule_set = GenericRelation('Rule') class Meta: ordering = ['name'] def get_absolute_url(self): return reverse('project-detail', kwargs={'pk': self.pk}) def __str__(self): return '{} » {}'.format(self.service, self.name)
[docs]class Farm(models.Model): name = models.CharField(max_length=128, validators=[validators.labelvalue]) source = models.CharField(max_length=128) class Meta: ordering = ['name'] unique_together = (('name', 'source',),) def get_absolute_url(self): return reverse('farm-detail', kwargs={'pk': self.pk}) def refresh(self): target = set() current = set(host.name for host in self.host_set.all()) for entry in plugins.discovery(): if self.source == entry.name: target.update(entry.load()().fetch(self.name)) remove = current - target add = target - current if add: Audit.log('Adding {} to {}'.format(add, self), self) Host.objects.bulk_create([ Host(name=name, farm_id=self.id) for name in add ]) if remove: Audit.log('Removing {} from {}'.format(add, self), self) Host.objects.filter(farm=self, name__in=remove).delete() return add, remove @classmethod def fetch(cls, source): for entry in plugins.discovery(): if entry.name == source: for farm in entry.load()().farms(): yield farm @cached_property def driver(self): '''Return configured driver for Farm model instance''' for entry in plugins.discovery(): if entry.name == self.source: return entry.load()() @property def editable(self): return not self.driver.remote
[docs] @classmethod def driver_set(cls): '''Return the list of drivers for Farm model''' for entry in plugins.discovery(): yield entry.name, entry.load()()
def __str__(self): return '{} ({})'.format(self.name, self.source)
[docs]class Host(models.Model): name = models.CharField(max_length=128) farm = models.ForeignKey('Farm', on_delete=models.CASCADE) class Meta: ordering = ['name'] unique_together = (('name', 'farm',),) def get_absolute_url(self): return reverse('host-detail', kwargs={'slug': self.name}) def __str__(self): return '{} [{}]'.format(self.name, self.farm.name)
class BaseExporter(models.Model): job = models.CharField( max_length=128, help_text="Exporter name. Example node, jmx, app" ) port = models.IntegerField(help_text="Port Exporter is running on") path = models.CharField( max_length=128, blank=True, help_text="Exporter path. Defaults to /metrics" ) scheme = models.CharField( max_length=5, choices=(("http", "http"), ("https", "https")), default="http", help_text="Scrape exporter over http or https", ) class Meta: abstract = True
[docs]class DefaultExporter(BaseExporter): class Meta: ordering = ["job", "port"] unique_together = (("job", "port", "path"),)
[docs]class Exporter(BaseExporter): project = models.ForeignKey("Project", on_delete=models.CASCADE) enabled = models.BooleanField(default=True) class Meta: ordering = ["job", "port"] unique_together = (("job", "port", "path", "scheme", "project"),) def __str__(self): return "{}:{}{}".format(self.job, self.port, self.path or "/metrics")
[docs]class Probe(models.Model): module = models.CharField(help_text='Probe Module from blackbox_exporter config', max_length=128, unique=True) description = models.TextField(blank=True) def __str__(self): return "{} » {}".format(self.module, self.description)
[docs]class URL(models.Model): url = models.URLField(max_length=256) project = models.ForeignKey("Project", on_delete=models.CASCADE) probe = models.ForeignKey("promgen.Probe", on_delete=models.CASCADE) class Meta: ordering = ["project__service", "project", "url"] def __str__(self): return "{} [{}]".format(self.project, self.url)
[docs]class Rule(models.Model): objects = ObjectFilterManager() name = models.CharField(max_length=128, unique=True, validators=[validators.metricname]) clause = models.TextField(help_text='Prometheus query') duration = models.CharField( max_length=128, validators=[validators.duration], help_text="Duration field with postfix. Example 30s, 5m, 1d" ) enabled = models.BooleanField(default=True) parent = models.ForeignKey( 'Rule', null=True, related_name='overrides', on_delete=models.SET_NULL ) content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, limit_choices_to=( models.Q(app_label='promgen', model='site') | models.Q(app_label='promgen', model='project') | models.Q(app_label='promgen', model='service')) ) object_id = models.PositiveIntegerField() content_object = GenericForeignKey('content_type', 'object_id', for_concrete_model=False) description = models.TextField(blank=True) class Meta: ordering = ['content_type', 'object_id', 'name'] @cached_property def labels(self): return {obj.name: obj.value for obj in self.rulelabel_set.all()} def add_label(self, name, value): return RuleLabel.objects.get_or_create(rule=self, name=name, value=value) def add_annotation(self, name, value): return RuleAnnotation.objects.get_or_create(rule=self, name=name, value=value) @cached_property def annotations(self): _annotations = {obj.name: obj.value for obj in self.ruleannotation_set.all()} # Skip when pk is not set, such as when test rendering a rule if self.pk and 'rule' not in _annotations: _annotations['rule'] = resolve_domain('rule-detail', pk=self.pk) return _annotations def __str__(self): return '{} [{}]'.format(self.name, self.content_object.name) def get_absolute_url(self): return reverse('rule-detail', kwargs={'pk': self.pk}) def set_object(self, content_type, object_id): self.content_type = ContentType.objects.get( model=content_type, app_label='promgen' ) self.object_id = object_id
[docs] def copy_to(self, content_type, object_id): ''' Make a copy under a new service It's important that we set pk to None so a new object is created, but we also need to ensure the new name is unique by appending some unique data to the end of the name ''' with transaction.atomic(): content_type = ContentType.objects.get(model=content_type, app_label='promgen') # First check to see if this rule is already overwritten for rule in Rule.objects.filter(parent_id=self.pk, content_type=content_type, object_id=object_id): return rule content_object = content_type.get_object_for_this_type(pk=object_id) orig_pk = self.pk self.pk = None self.parent_id = orig_pk self.name = '{}_{}'.format(self.name, slugify(content_object.name)).replace('-', '_') self.content_type = content_type self.object_id = object_id self.enabled = False self.clause = self.clause.replace(macro.EXCLUSION_MACRO, '{}="{}",{}'.format( content_type.model, content_object.name, macro.EXCLUSION_MACRO )) self.save() # Add a label to our new rule by default, to help ensure notifications # get routed to the notifier we expect self.add_label(content_type.model, content_object.name) for label in RuleLabel.objects.filter(rule_id=orig_pk): # Skip service labels from our previous rule if label.name in ['service', 'project']: logger.debug('Skipping %s: %s', label.name, label.value) continue logger.debug('Copying %s to %s', label, self) label.pk = None label.rule = self label.save() for annotation in RuleAnnotation.objects.filter(rule_id=orig_pk): logger.debug('Copying %s to %s', annotation, self) annotation.pk = None annotation.rule = self annotation.save() return self
[docs]class RuleLabel(models.Model): name = models.CharField(max_length=128) value = models.CharField(max_length=128) rule = models.ForeignKey('Rule', on_delete=models.CASCADE)
[docs]class RuleAnnotation(models.Model): name = models.CharField(max_length=128) value = models.TextField() rule = models.ForeignKey('Rule', on_delete=models.CASCADE)
[docs]class AlertLabel(models.Model): alert = models.ForeignKey('Alert', on_delete=models.CASCADE) name = models.CharField(max_length=128) value = models.TextField()
[docs]class Alert(models.Model): created = models.DateTimeField(default=timezone.now) body = models.TextField() sent_count = models.PositiveIntegerField(default=0) error_count = models.PositiveIntegerField(default=0) def get_absolute_url(self): return reverse("alert-detail", kwargs={"pk": self.pk}) def expand(self): # Map of Prometheus labels to Promgen objects LABEL_MAPPING = [ ('project', Project), ('service', Service), ] routable = {} data = json.loads(self.body) data.setdefault('commonLabels', {}) data.setdefault('commonAnnotations', {}) # Set our link back to Promgen for processed notifications # The original externalURL can still be visible from the alerts page data['externalURL'] = resolve_domain(self.get_absolute_url()) # Look through our labels and find the object from Promgen's DB # If we find an object in Promgen, add an annotation with a direct link for label, klass in LABEL_MAPPING: if label not in data['commonLabels']: logger.debug('Missing label %s', label) continue # Should only find a single value, but I think filter is a little # bit more forgiving than get in terms of throwing errors for obj in klass.objects.filter(name=data['commonLabels'][label]): logger.debug('Found %s %s', label, obj) routable[label] = obj data['commonAnnotations'][label] = resolve_domain(obj) return routable, data @cached_property def json(self): return json.loads(self.body)
[docs]class AlertError(models.Model): alert = models.ForeignKey(Alert, on_delete=models.CASCADE) created = models.DateTimeField(default=timezone.now) message = models.TextField()
[docs]class Audit(models.Model): body = models.TextField() created = models.DateTimeField() data = models.TextField(blank=True) old = models.TextField(blank=True) content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, null=True) object_id = models.PositiveIntegerField(default=0) content_object = GenericForeignKey('content_type', 'object_id') user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True, default=None) @property def hilight(self): if self.body.startswith('Created'): return 'success' if self.body.startswith('Updated'): return 'warning' if self.body.startswith('Deleted'): return 'danger' return '' @classmethod def log(cls, body, instance=None, old=None, **kwargs): from promgen.middleware import get_current_user kwargs['body'] = body kwargs['created'] = timezone.now() kwargs['user'] = get_current_user() if instance: kwargs['content_type'] = ContentType.objects.get_for_model(instance) kwargs['object_id'] = instance.id kwargs['data'] = json.dumps(model_to_dict(instance), sort_keys=True) if old: kwargs['old'] = json.dumps(model_to_dict(old), sort_keys=True) return cls.objects.create(**kwargs)
[docs]class Prometheus(models.Model): shard = models.ForeignKey('promgen.Shard', on_delete=models.CASCADE) host = models.CharField(max_length=128) port = models.IntegerField() def __str__(self): return '{}:{}'.format(self.host, self.port) class Meta: ordering = ['shard', 'host'] unique_together = (('host', 'port',),) verbose_name_plural = 'prometheis'