From 09524173ed76a92a34645755503a628aace24f97 Mon Sep 17 00:00:00 2001 From: wouter Date: Mon, 23 Jun 2025 11:16:54 +0200 Subject: [PATCH 1/2] update metadata --- .travis.yml | 4 +- example_sysmetrics.py | 15 +- pyformance/__init__.py | 52 +++-- pyformance/meters/__init__.py | 25 +- pyformance/meters/counter.py | 28 ++- pyformance/meters/gauge.py | 50 ++-- pyformance/meters/histogram.py | 64 ++++-- pyformance/meters/meter.py | 43 ++-- pyformance/meters/timer.py | 102 +++++---- pyformance/registry.py | 151 +++++++------ pyformance/reporters/carbon_reporter.py | 9 +- pyformance/reporters/reporter.py | 5 +- pyformance/reporters/syslog_reporter.py | 5 +- pyformance/stats/__init__.py | 23 +- pyformance/stats/moving_average.py | 34 ++- pyformance/stats/samples.py | 75 ++++-- pyformance/stats/snapshot.py | 49 ++-- setup.py | 10 +- tests/test__carbon_reporter.py | 7 +- tests/test__counter.py | 49 ++-- tests/test__gauge.py | 37 +-- tests/test__histogram.py | 140 +++++++----- tests/test__meter.py | 163 ++++++------- tests/test__moving_average.py | 289 +++++++++++++----------- tests/test__timer.py | 67 ++++-- tox.ini | 2 +- 26 files changed, 917 insertions(+), 581 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6b77cb2..285d6ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,6 @@ language: python python: - - 2.7 - - 3.4 - - 3.6 + - 3.12 # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors install: pip install -e . # command to run tests, e.g. python setup.py test diff --git a/example_sysmetrics.py b/example_sysmetrics.py index b033025..029766f 100644 --- a/example_sysmetrics.py +++ b/example_sysmetrics.py @@ -5,7 +5,6 @@ import time import json -import six import psutil from pyformance import global_registry @@ -21,16 +20,16 @@ def __init__(self, registry=None): def collect_disk_io(self, whitelist=[]): stats = psutil.disk_io_counters(perdisk=True) - for entry, stat in six.iteritems(stats): + for entry, stat in stats.items(): if not whitelist or entry in whitelist: - for k, v in six.iteritems(stat._asdict()): + for k, v in stat._asdict().items(): self.registry.gauge("disk-%s.%s" % (entry, k)).set_value(v) def collect_network_io(self, whitelist=[]): stats = psutil.net_io_counters(pernic=True) - for entry, stat in six.iteritems(stats): + for entry, stat in stats.items(): if not whitelist or entry in whitelist: - for k, v in six.iteritems(stat._asdict()): + for k, v in stat._asdict().items(): self.registry.gauge( "nic-%s.%s" % (entry.replace(" ", "_"), k) ).set_value(v) @@ -39,17 +38,17 @@ def collect_cpu_times(self, whitelist=[]): stats = psutil.cpu_times(percpu=True) for entry, stat in enumerate(stats): if not whitelist or entry in whitelist: - for k, v in six.iteritems(stat._asdict()): + for k, v in stat._asdict().items(): self.registry.gauge("cpu%d.%s" % (entry, k)).set_value(v) def collect_swap_usage(self): stats = psutil.swap_memory() - for k, v in six.iteritems(stats._asdict()): + for k, v in stats._asdict().items(): self.registry.gauge("swap.%s" % k).set_value(v) def collect_virtmem_usage(self): stats = psutil.virtual_memory() - for k, v in six.iteritems(stats._asdict()): + for k, v in stats._asdict().items(): self.registry.gauge("virtmem.%s" % k).set_value(v) def collect_uptime(self): diff --git a/pyformance/__init__.py b/pyformance/__init__.py index 7f48882..e34d100 100644 --- a/pyformance/__init__.py +++ b/pyformance/__init__.py @@ -1,13 +1,39 @@ -__import__("pkg_resources").declare_namespace(__name__) - -from .registry import MetricsRegistry, global_registry, set_global_registry -from .registry import timer, counter, meter, histogram, gauge -from .registry import ( - dump_metrics, - clear, - count_calls, - meter_calls, - hist_calls, - time_calls, -) -from .meters.timer import call_too_long +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from typing import Protocol + + +class Clock(Protocol): + + def time(self) -> float: ... + + +from .registry import MetricsRegistry as MetricsRegistry # noqa F401 +from .registry import clear as clear # noqa F401 +from .registry import count_calls as count_calls # noqa F401 +from .registry import counter as counter # noqa F401 +from .registry import dump_metrics as dump_metrics # noqa F401 +from .registry import gauge as gauge # noqa F401 +from .registry import global_registry as global_registry # noqa F401 +from .registry import hist_calls as hist_calls # noqa F401 +from .registry import histogram as histogram # noqa F401 +from .registry import meter as meter # noqa F401 +from .registry import meter_calls as meter_calls # noqa F401 +from .registry import set_global_registry as set_global_registry # noqa F401 +from .registry import time_calls as time_calls # noqa F401 +from .registry import timer as timer # noqa F401 diff --git a/pyformance/meters/__init__.py b/pyformance/meters/__init__.py index fd86465..b0bc02e 100644 --- a/pyformance/meters/__init__.py +++ b/pyformance/meters/__init__.py @@ -1,5 +1,26 @@ +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + from .counter import Counter -from .meter import Meter +from .gauge import CallbackGauge, Gauge, SimpleGauge from .histogram import Histogram +from .meter import Meter from .timer import Timer -from .gauge import Gauge, CallbackGauge, SimpleGauge + +__all__ = ["Counter", "CallbackGauge", "Gauge", "SimpleGauge", "Histogram", "Meter", "Timer"] + +type any_meter = Histogram | Meter | Gauge[int | float] | Timer | Counter diff --git a/pyformance/meters/counter.py b/pyformance/meters/counter.py index 5c77742..1e9c6e1 100644 --- a/pyformance/meters/counter.py +++ b/pyformance/meters/counter.py @@ -1,31 +1,47 @@ +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + from threading import Lock class Counter(object): - """ An incrementing and decrementing metric """ - def __init__(self): + def __init__(self) -> None: super(Counter, self).__init__() self.lock = Lock() self.counter = 0 - def inc(self, val=1): + def inc(self, val: int = 1) -> None: "increment counter by val (default is 1)" with self.lock: self.counter = self.counter + val - def dec(self, val=1): + def dec(self, val: int = 1) -> None: "decrement counter by val (default is 1)" self.inc(-val) - def get_count(self): + def get_count(self) -> int: "return current value of counter" return self.counter - def clear(self): + def clear(self) -> None: "reset counter to 0" with self.lock: self.counter = 0 diff --git a/pyformance/meters/gauge.py b/pyformance/meters/gauge.py index 75c6f89..7dad9f1 100644 --- a/pyformance/meters/gauge.py +++ b/pyformance/meters/gauge.py @@ -1,57 +1,77 @@ -class Gauge(object): +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from typing import Callable, Union + + +class Gauge[T: Union[int, float]]: """ A base class for reading of a particular. - + For example, to instrument a queue depth: - + class QueueLengthGaguge(Gauge): def __init__(self, queue): super(QueueGaguge, self).__init__() self.queue = queue - + def get_value(self): return len(self.queue) - + """ - def get_value(self): + def get_value(self) -> T: "A subclass of Gauge should implement this method" raise NotImplementedError() -class CallbackGauge(Gauge): - +class CallbackGauge[T: Union[int, float]](Gauge[T]): """ A Gauge reading for a given callback """ - def __init__(self, callback): + def __init__(self, callback: Callable[[], T]) -> None: "constructor expects a callable" super(CallbackGauge, self).__init__() self.callback = callback - def get_value(self): + def get_value(self) -> T: "returns the result of callback which is executed each time" return self.callback() -class SimpleGauge(Gauge): - +class SimpleGauge[T: Union[int, float]](Gauge[T]): """ A gauge which holds values with simple getter- and setter-interface """ - def __init__(self, value=float("nan")): + def __init__(self, value: T) -> None: "constructor accepts initial value" super(SimpleGauge, self).__init__() self._value = value - def get_value(self): + def get_value(self) -> T: "getter returns current value" return self._value - def set_value(self, value): + def set_value(self, value: T) -> None: "setter changes current value" # XXX: add locking? self._value = value + + +type AnyGauge = Gauge[float | int] diff --git a/pyformance/meters/histogram.py b/pyformance/meters/histogram.py index 9f7b93a..08565b6 100644 --- a/pyformance/meters/histogram.py +++ b/pyformance/meters/histogram.py @@ -1,16 +1,44 @@ -import time +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + import math +import time from threading import Lock -from ..stats.samples import ExpDecayingSample, DEFAULT_SIZE, DEFAULT_ALPHA +from .. import Clock +from ..stats.samples import DEFAULT_ALPHA, DEFAULT_SIZE, ExpDecayingSample, Sample +from ..stats.snapshot import Snapshot -class Histogram(object): +class Histogram(object): """ A metric which calculates the distribution of a value. """ - def __init__(self, size=DEFAULT_SIZE, alpha=DEFAULT_ALPHA, clock=time, sample=None): + counter: float # Would be expected to be int? + max: float + min: float + sum: float + var: tuple[float, float] + sample: Sample + + def __init__( + self, size: int = DEFAULT_SIZE, alpha: float = DEFAULT_ALPHA, clock: Clock = time, sample: Sample | None = None + ) -> None: """ Creates a new instance of a L{Histogram}. """ @@ -22,7 +50,7 @@ def __init__(self, size=DEFAULT_SIZE, alpha=DEFAULT_ALPHA, clock=time, sample=No self.sample = sample self.clear() - def add(self, value): + def add(self, value: float) -> None: """ Add value to histogram @@ -36,7 +64,7 @@ def add(self, value): self.sum = self.sum + value self._update_var(value) - def clear(self): + def clear(self) -> None: "reset histogram to initial state" with self.lock: self.sample.clear() @@ -44,52 +72,52 @@ def clear(self): self.max = -2147483647.0 self.min = 2147483647.0 self.sum = 0.0 - self.var = [-1.0, 0.0] + self.var = (-1.0, 0.0) - def get_count(self): + def get_count(self) -> float: "get current value of counter" return self.counter - def get_sum(self): + def get_sum(self) -> float: "get current sum" return self.sum - def get_max(self): + def get_max(self) -> float: "get current maximum" return self.max - def get_min(self): + def get_min(self) -> float: "get current minimum" return self.min - def get_mean(self): + def get_mean(self) -> float: "get current mean" if self.counter > 0: return self.sum / self.counter return 0 - def get_stddev(self): + def get_stddev(self) -> float: "get current standard deviation" if self.counter > 0: return math.sqrt(self.get_var()) return 0 - def get_var(self): + def get_var(self) -> float: "get current variance" if self.counter > 1: return self.var[1] / (self.counter - 1) return 0 - def get_snapshot(self): + def get_snapshot(self) -> Snapshot: "get snapshot instance which holds the percentiles" return self.sample.get_snapshot() - def _update_var(self, value): + def _update_var(self, value: float) -> None: old_m, old_s = self.var - new_m, new_s = [0.0, 0.0] + new_m, new_s = (0.0, 0.0) if old_m == -1: new_m = value else: new_m = old_m + ((value - old_m) / self.counter) new_s = old_s + ((value - old_m) * (value - new_m)) - self.var = [new_m, new_s] + self.var = (new_m, new_s) diff --git a/pyformance/meters/meter.py b/pyformance/meters/meter.py index 3a6b9bc..66a2576 100644 --- a/pyformance/meters/meter.py +++ b/pyformance/meters/meter.py @@ -1,22 +1,40 @@ +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + import time from threading import Lock + +from .. import Clock from ..stats.moving_average import ExpWeightedMovingAvg class Meter(object): - """ A meter metric which measures mean throughput and one-, five-, and fifteen-minute exponentially-weighted moving average throughputs. """ - def __init__(self, clock=time): + def __init__(self, clock: Clock = time) -> None: super(Meter, self).__init__() self.lock = Lock() self.clock = clock self.clear() - def clear(self): + def clear(self) -> None: with self.lock: self.start_time = self.clock.time() self.counter = 0.0 @@ -24,35 +42,32 @@ def clear(self): self.m5rate = ExpWeightedMovingAvg(period=5, clock=self.clock) self.m15rate = ExpWeightedMovingAvg(period=15, clock=self.clock) - def get_one_minute_rate(self): + def get_one_minute_rate(self) -> float: return self.m1rate.get_rate() - def get_five_minute_rate(self): + def get_five_minute_rate(self) -> float: return self.m5rate.get_rate() - def get_fifteen_minute_rate(self): + def get_fifteen_minute_rate(self) -> float: return self.m15rate.get_rate() - def tick(self): + def tick(self) -> None: self.m1rate.tick() self.m5rate.tick() self.m15rate.tick() - def mark(self, value=1): + def mark(self, value: float = 1) -> None: with self.lock: self.counter += value self.m1rate.add(value) self.m5rate.add(value) self.m15rate.add(value) - def get_count(self): + def get_count(self) -> float: return self.counter - def get_mean_rate(self): + def get_mean_rate(self) -> float: if self.counter == 0: return 0 - elapsed = self.clock.time() - self.start_time + elapsed: float = self.clock.time() - self.start_time return self.counter / elapsed - - def _convertNsRate(self, ratePerNs): - return ratePerNs diff --git a/pyformance/meters/timer.py b/pyformance/meters/timer.py index 5d5aa57..15efd1d 100644 --- a/pyformance/meters/timer.py +++ b/pyformance/meters/timer.py @@ -1,21 +1,41 @@ +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import abc import time +from types import TracebackType +from typing import Optional, Type + +import pyformance +from pyformance.meters.histogram import Histogram +from pyformance.stats.samples import DEFAULT_ALPHA, DEFAULT_SIZE, Sample -try: - from blinker import Namespace -except ImportError: - Namespace = None -from .histogram import Histogram, DEFAULT_SIZE, DEFAULT_ALPHA +from .. import Clock from .meter import Meter -if Namespace is not None: - timer_signals = Namespace() - call_too_long = timer_signals.signal("call_too_long") -else: - call_too_long = None +class TimerSink(abc.ABC): -class Timer(object): + @abc.abstractmethod + def add(self, value: float) -> None: + pass + +class Timer(object): """ A timer metric which aggregates timing durations and provides duration statistics, plus throughput statistics via Meter and Histogram. @@ -24,75 +44,73 @@ class Timer(object): def __init__( self, - threshold=None, - size=DEFAULT_SIZE, - alpha=DEFAULT_ALPHA, - clock=time, - sink=None, - sample=None, - ): + size: int = DEFAULT_SIZE, + alpha: float = DEFAULT_ALPHA, + clock: Clock = time, + sink: TimerSink | None = None, + sample: Sample | None = None, + ) -> None: super(Timer, self).__init__() self.meter = Meter(clock=clock) self.hist = Histogram(size=size, alpha=alpha, clock=clock, sample=sample) self.sink = sink - self.threshold = threshold - def get_count(self): + def get_count(self) -> float: "get count from internal histogram" return self.hist.get_count() - def get_sum(self): + def get_sum(self) -> float: "get sum from snapshot of internal histogram" return self.get_snapshot().get_sum() - def get_max(self): + def get_max(self) -> float: "get max from snapshot of internal histogram" return self.get_snapshot().get_max() - def get_min(self): + def get_min(self) -> float: "get min from snapshot of internal histogram" return self.get_snapshot().get_min() - def get_mean(self): + def get_mean(self) -> float: "get mean from snapshot of internal histogram" return self.get_snapshot().get_mean() - def get_stddev(self): + def get_stddev(self) -> float: "get stddev from snapshot of internal histogram" return self.get_snapshot().get_stddev() - def get_var(self): + def get_var(self) -> float: "get var from snapshot of internal histogram" return self.get_snapshot().get_var() - def get_snapshot(self): + def get_snapshot(self) -> pyformance.stats.snapshot.Snapshot: "get snapshot from internal histogram" return self.hist.get_snapshot() - def get_mean_rate(self): + def get_mean_rate(self) -> float: "get mean rate from internal meter" return self.meter.get_mean_rate() - def get_one_minute_rate(self): + def get_one_minute_rate(self) -> float: "get 1 minut rate from internal meter" return self.meter.get_one_minute_rate() - def get_five_minute_rate(self): + def get_five_minute_rate(self) -> float: "get 5 minute rate from internal meter" return self.meter.get_five_minute_rate() - def get_fifteen_minute_rate(self): + def get_fifteen_minute_rate(self) -> float: "get 15 rate from internal meter" return self.meter.get_fifteen_minute_rate() - def _update(self, seconds): + def _update(self, seconds: float) -> None: if seconds >= 0: self.hist.add(seconds) self.meter.mark() if self.sink: self.sink.add(seconds) - def time(self, *args, **kwargs): + def time(self, *args: object, **kwargs: object) -> "TimerContext": """ Parameters will be sent to signal, if fired. Returns a timer context instance which can be used from a with-statement. @@ -100,14 +118,14 @@ def time(self, *args, **kwargs): """ return TimerContext(self, self.meter.clock, *args, **kwargs) - def clear(self): + def clear(self) -> None: "clear internal histogram and meter" self.hist.clear() self.meter.clear() class TimerContext(object): - def __init__(self, timer, clock, *args, **kwargs): + def __init__(self, timer: Timer, clock: Clock, *args: object, **kwargs: object) -> None: super(TimerContext, self).__init__() self.clock = clock self.timer = timer @@ -115,19 +133,13 @@ def __init__(self, timer, clock, *args, **kwargs): self.kwargs = kwargs self.args = args - def stop(self): - elapsed = self.clock.time() - self.start_time + def stop(self) -> float: + elapsed: float = self.clock.time() - self.start_time self.timer._update(elapsed) - if ( - self.timer.threshold - and self.timer.threshold < elapsed - and call_too_long is not None - ): - call_too_long.send(self.timer, elapsed=elapsed, *self.args, **self.kwargs) return elapsed - def __enter__(self): + def __enter__(self) -> None: pass - def __exit__(self, t, v, tb): + def __exit__(self, t: Optional[Type[BaseException]], v: Optional[BaseException], tb: Optional[TracebackType]) -> None: self.stop() diff --git a/pyformance/registry.py b/pyformance/registry.py index 5f6d24a..65ced43 100644 --- a/pyformance/registry.py +++ b/pyformance/registry.py @@ -1,12 +1,34 @@ +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + import functools import re import time -import sys -from .meters import Counter, Histogram, Meter, Timer, Gauge, CallbackGauge, SimpleGauge +from typing import Callable, Mapping, Optional, Union +from . import Clock +from .meters import CallbackGauge, Counter, Gauge, Histogram, Meter, SimpleGauge, Timer, any_meter +from .meters.gauge import AnyGauge +from .meters.timer import TimerSink -class MetricsRegistry(object): +type serialized_meter = Mapping[str, str | int | float] + +class MetricsRegistry: """ A single interface used to gather metrics on a service. It keeps track of all the relevant Counters, Meters, Histograms, and Timers. It does not have @@ -14,18 +36,18 @@ class MetricsRegistry(object): L{MetricsRegistry} to manage all of its metrics tools. """ - def __init__(self, clock=time): + def __init__(self, clock: Clock = time) -> None: """ Creates a new L{MetricsRegistry} instance. """ - self._timers = {} - self._meters = {} - self._counters = {} - self._histograms = {} - self._gauges = {} + self._timers: dict[str, Timer] = {} + self._meters: dict[str, Meter] = {} + self._counters: dict[str, Counter] = {} + self._histograms: dict[str, Histogram] = {} + self._gauges: dict[str, Gauge[Union[int, float]]] = {} self._clock = clock - def add(self, key, metric): + def add(self, key: str, metric: any_meter) -> None: """ Use this method to manually add custom metric instances to the registry which are not created with their constructor's default arguments, @@ -50,7 +72,7 @@ def add(self, key, metric): return raise TypeError("Invalid class. Could not register metric %r" % key) - def counter(self, key): + def counter(self, key: str) -> Counter: """ Gets a counter based on a key, creates a new one if it does not exist. @@ -63,7 +85,7 @@ def counter(self, key): self._counters[key] = Counter() return self._counters[key] - def histogram(self, key): + def histogram(self, key: str) -> Histogram: """ Gets a histogram based on a key, creates a new one if it does not exist. @@ -76,20 +98,23 @@ def histogram(self, key): self._histograms[key] = Histogram(clock=self._clock) return self._histograms[key] - def gauge(self, key, gauge=None, default=float("nan")): + def gauge[T: float | int]( + self, key: str, gauge: Gauge[T] | Callable[[], T] | None = None, default: float = float("nan") + ) -> AnyGauge: + out: AnyGauge if key not in self._gauges: if gauge is None: - gauge = SimpleGauge( - default - ) # raise TypeError("gauge required for registering") + out = SimpleGauge(default) # raise TypeError("gauge required for registering") elif not isinstance(gauge, Gauge): if not callable(gauge): raise TypeError("gauge getter not callable") - gauge = CallbackGauge(gauge) - self._gauges[key] = gauge + out = CallbackGauge(gauge) + else: + out = gauge + self._gauges[key] = out return self._gauges[key] - def meter(self, key): + def meter(self, key: str) -> Meter: """ Gets a meter based on a key, creates a new one if it does not exist. @@ -102,10 +127,10 @@ def meter(self, key): self._meters[key] = Meter(clock=self._clock) return self._meters[key] - def create_sink(self): + def create_sink(self) -> TimerSink | None: return None - def timer(self, key): + def timer(self, key: str) -> Timer: """ Gets a timer based on a key, creates a new one if it does not exist. @@ -118,26 +143,26 @@ def timer(self, key): self._timers[key] = Timer(clock=self._clock, sink=self.create_sink()) return self._timers[key] - def clear(self): + def clear(self) -> None: self._meters.clear() self._counters.clear() self._gauges.clear() self._timers.clear() self._histograms.clear() - def _get_counter_metrics(self, key): + def _get_counter_metrics(self, key: str) -> serialized_meter: if key in self._counters: counter = self._counters[key] return {"count": counter.get_count()} return {} - def _get_gauge_metrics(self, key): + def _get_gauge_metrics(self, key: str) -> serialized_meter: if key in self._gauges: gauge = self._gauges[key] return {"value": gauge.get_value()} return {} - def _get_histogram_metrics(self, key): + def _get_histogram_metrics(self, key: str) -> serialized_meter: if key in self._histograms: histogram = self._histograms[key] snapshot = histogram.get_snapshot() @@ -155,7 +180,7 @@ def _get_histogram_metrics(self, key): return res return {} - def _get_meter_metrics(self, key): + def _get_meter_metrics(self, key: str) -> serialized_meter: if key in self._meters: meter = self._meters[key] res = { @@ -168,7 +193,7 @@ def _get_meter_metrics(self, key): return res return {} - def _get_timer_metrics(self, key): + def _get_timer_metrics(self, key: str) -> serialized_meter: if key in self._timers: timer = self._timers[key] snapshot = timer.get_snapshot() @@ -192,7 +217,7 @@ def _get_timer_metrics(self, key): return res return {} - def get_metrics(self, key): + def get_metrics(self, key: str) -> serialized_meter: """ Gets all the metrics for a specified key. @@ -201,7 +226,8 @@ def get_metrics(self, key): :return: C{dict} """ - metrics = {} + metrics: dict[str, str | int | float] = {} + getter: Callable[[str], serialized_meter] for getter in ( self._get_counter_metrics, self._get_histogram_metrics, @@ -212,13 +238,13 @@ def get_metrics(self, key): metrics.update(getter(key)) return metrics - def dump_metrics(self): + def dump_metrics(self) -> Mapping[str, serialized_meter]: """ Formats all of the metrics and returns them as a dict. :return: C{list} of C{dict} of metrics """ - metrics = {} + metrics: dict[str, serialized_meter] = {} for metric_type in ( self._counters, self._histograms, @@ -233,8 +259,7 @@ def dump_metrics(self): class RegexRegistry(MetricsRegistry): - - """ + r""" A single interface used to gather metrics on a service. This class uses a regex to combine measures that match a pattern. For example, if you have a REST API, instead of defining a timer for each method, you can use a regex to capture all API calls and group them. @@ -244,81 +269,81 @@ class RegexRegistry(MetricsRegistry): /api/users/2/edit -> users/edit """ - def __init__(self, pattern=None, clock=time): + def __init__(self, pattern: Optional[str] = None, clock: Clock = time) -> None: super(RegexRegistry, self).__init__(clock) if pattern is not None: self.pattern = re.compile(pattern) else: self.pattern = re.compile("^$") - def _get_key(self, key): + def _get_key(self, key: str) -> str: matches = self.pattern.finditer(key) key = "/".join((v for match in matches for v in match.groups() if v)) return key - def timer(self, key): + def timer(self, key: str) -> Timer: return super(RegexRegistry, self).timer(self._get_key(key)) - def histogram(self, key): + def histogram(self, key: str) -> Histogram: return super(RegexRegistry, self).histogram(self._get_key(key)) - def counter(self, key): + def counter(self, key: str) -> Counter: return super(RegexRegistry, self).counter(self._get_key(key)) - def gauge(self, key, gauge=None, default=float("nan")): + def gauge[T: float | int]( + self, key: str, gauge: Gauge[T] | Callable[[], T] | None = None, default: float = float("nan") + ) -> AnyGauge: return super(RegexRegistry, self).gauge(self._get_key(key), gauge, default) - def meter(self, key): + def meter(self, key: str) -> Meter: return super(RegexRegistry, self).meter(self._get_key(key)) _global_registry = MetricsRegistry() -def global_registry(): +def global_registry() -> MetricsRegistry: return _global_registry -def set_global_registry(registry): +def set_global_registry(registry: MetricsRegistry) -> None: global _global_registry _global_registry = registry -def counter(key): +def counter(key: str) -> Counter: return _global_registry.counter(key) -def histogram(key): +def histogram(key: str) -> Histogram: return _global_registry.histogram(key) -def meter(key): +def meter(key: str) -> Meter: return _global_registry.meter(key) -def timer(key): +def timer(key: str) -> Timer: return _global_registry.timer(key) -def gauge(key, gauge=None): +def gauge(key: str, gauge: AnyGauge | None = None) -> AnyGauge: return _global_registry.gauge(key, gauge) -def dump_metrics(): +def dump_metrics() -> Mapping[str, serialized_meter]: return _global_registry.dump_metrics() -def clear(): +def clear() -> None: return _global_registry.clear() -def get_qualname(obj): - if sys.version_info[0] > 2: - return obj.__qualname__ - return obj.__name__ +def get_qualname[**P, R](obj: Callable[P, R]) -> str: + return obj.__qualname__ -def count_calls(fn): +def count_calls[**P, R](fn: Callable[P, R]) -> Callable[P, R]: """ Decorator to track the number of times a function is called. @@ -330,14 +355,14 @@ def count_calls(fn): """ @functools.wraps(fn) - def wrapper(*args, **kwargs): + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: counter("%s_calls" % get_qualname(fn)).inc() return fn(*args, **kwargs) return wrapper -def meter_calls(fn): +def meter_calls[**P, R](fn: Callable[P, R]) -> Callable[P, R]: """ Decorator to the rate at which a function is called. @@ -349,14 +374,14 @@ def meter_calls(fn): """ @functools.wraps(fn) - def wrapper(*args, **kwargs): + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: meter("%s_calls" % get_qualname(fn)).mark() return fn(*args, **kwargs) return wrapper -def hist_calls(fn): +def hist_calls[**P, R](fn: Callable[P, R]) -> Callable[P, R]: """ Decorator to check the distribution of return values of a function. @@ -368,17 +393,17 @@ def hist_calls(fn): """ @functools.wraps(fn) - def wrapper(*args, **kwargs): + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: _histogram = histogram("%s_calls" % get_qualname(fn)) rtn = fn(*args, **kwargs) - if type(rtn) in (int, float): - _histogram.update(rtn) + if isinstance(rtn, (int, float)): + _histogram.add(rtn) return rtn return wrapper -def time_calls(fn): +def time_calls[**P, R](fn: Callable[P, R]) -> Callable[P, R]: """ Decorator to time the execution of the function. @@ -390,7 +415,7 @@ def time_calls(fn): """ @functools.wraps(fn) - def wrapper(*args, **kwargs): + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: _timer = timer("%s_calls" % get_qualname(fn)) with _timer.time(fn=get_qualname(fn)): return fn(*args, **kwargs) diff --git a/pyformance/reporters/carbon_reporter.py b/pyformance/reporters/carbon_reporter.py index ad5402b..b74d59b 100644 --- a/pyformance/reporters/carbon_reporter.py +++ b/pyformance/reporters/carbon_reporter.py @@ -4,7 +4,6 @@ import struct import pickle import contextlib -from six import iteritems from .reporter import Reporter @@ -54,8 +53,8 @@ def _collect_metrics(self, registry, timestamp=None): "%s%s.%s" % (self.prefix, metric_name, metric_key), (timestamp, metric_value), ) - for metric_name, metric in iteritems(metrics) - for metric_key, metric_value in iteritems(metric) + for metric_name, metric in metrics.items() + for metric_key, metric_value in metric.items() ], protocol=2, ) @@ -63,8 +62,8 @@ def _collect_metrics(self, registry, timestamp=None): return header + payload else: metrics_data = [] - for metric_name, metric in iteritems(metrics): - for metric_key, metric_value in iteritems(metric): + for metric_name, metric in metrics.items(): + for metric_key, metric_value in metric.items(): metric_line = "%s%s.%s %s %s\n" % ( self.prefix, metric_name, diff --git a/pyformance/reporters/reporter.py b/pyformance/reporters/reporter.py index d2a4b62..118f830 100644 --- a/pyformance/reporters/reporter.py +++ b/pyformance/reporters/reporter.py @@ -1,6 +1,5 @@ import time from threading import Thread, Event -import six from ..registry import global_registry, get_qualname @@ -47,9 +46,7 @@ def _loop(self): pass next_loop_time += self.reporting_interval wait = max(0, next_loop_time - time.time()) - if six.PY2: - time.sleep(wait) - elif self._stopped.wait(timeout=wait): + if self._stopped.wait(timeout=wait): # wait is faster/better in Python 3 # See http://stackoverflow.com/questions/29082268/python-time-sleep-vs-event-wait break # true if timeout diff --git a/pyformance/reporters/syslog_reporter.py b/pyformance/reporters/syslog_reporter.py index 3311029..9a208dd 100644 --- a/pyformance/reporters/syslog_reporter.py +++ b/pyformance/reporters/syslog_reporter.py @@ -3,7 +3,6 @@ import socket import logging import logging.handlers -from six import iteritems import json from .reporter import Reporter @@ -58,8 +57,8 @@ def _collect_metrics(self, registry, timestamp=None): metrics_data = {"timestamp": timestamp} metrics = registry.dump_metrics() - for metric_name, metric in iteritems(metrics): - for metric_key, metric_value in iteritems(metric): + for metric_name, metric in metrics.items(): + for metric_key, metric_value in metric.items(): metrics_data["{}.{}".format(metric_name, metric_key)] = metric_value result = json.dumps(metrics_data, sort_keys=True) return result diff --git a/pyformance/stats/__init__.py b/pyformance/stats/__init__.py index 486f658..2034f41 100644 --- a/pyformance/stats/__init__.py +++ b/pyformance/stats/__init__.py @@ -1,3 +1,20 @@ -from .samples import ExpDecayingSample -from .moving_average import ExpWeightedMovingAvg -from .snapshot import Snapshot +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from .moving_average import ExpWeightedMovingAvg # noqa F401 +from .samples import ExpDecayingSample # noqa F401 +from .snapshot import Snapshot # noqa F401 diff --git a/pyformance/stats/moving_average.py b/pyformance/stats/moving_average.py index 4ac6eaa..846c3a9 100644 --- a/pyformance/stats/moving_average.py +++ b/pyformance/stats/moving_average.py @@ -1,9 +1,27 @@ +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + import math import time +from pyformance import Clock -class ExpWeightedMovingAvg(object): +class ExpWeightedMovingAvg(object): """ An exponentially-weighted moving average. """ @@ -11,7 +29,7 @@ class ExpWeightedMovingAvg(object): INTERVAL = 5.0 # seconds SECONDS_PER_MINUTE = 60.0 - def __init__(self, period, interval=INTERVAL, clock=time): + def __init__(self, period: int, interval: float = INTERVAL, clock: Clock = time) -> None: """ Create a new EWMA with a specific smoothing constant. @@ -24,21 +42,21 @@ def __init__(self, period, interval=INTERVAL, clock=time): self.clock = clock self.uncounted = 0.0 self.interval = interval - self.rate = -1 + self.rate: float = -1.0 self.period = period * ExpWeightedMovingAvg.SECONDS_PER_MINUTE self.last_tick = self.clock.time() - def get_rate(self): + def get_rate(self) -> float: if self.clock.time() - self.last_tick >= self.interval: self.tick() if self.rate >= 0: return self.rate return 0 - def add(self, value): + def add(self, value: float) -> None: self.uncounted += value - def tick(self): + def tick(self) -> None: """ Mark the passage of time and decay the current rate accordingly. """ @@ -58,10 +76,10 @@ def tick(self): self.last_tick = now - def _alpha(self, interval): + def _alpha(self, interval: float) -> float: """ Calculate the alpha based on the time since the last tick. This is - necessary because a single threaded Python program loses precision + necessary because a single threaded Python program loses precision under high load, so we can't assume a consistant I{EWMA._interval}. :type interval: C{float} diff --git a/pyformance/stats/samples.py b/pyformance/stats/samples.py index fa3515d..d573aa1 100644 --- a/pyformance/stats/samples.py +++ b/pyformance/stats/samples.py @@ -1,15 +1,45 @@ -import time -import random -import math +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + import heapq +import math +import random +import time + +from .. import Clock from .snapshot import Snapshot DEFAULT_SIZE = 1028 DEFAULT_ALPHA = 0.015 -class ExpDecayingSample(object): +# TODO: do I ABC this, may cpst us a few % performance?? +class Sample: + def clear(self) -> None: + raise NotImplementedError() + + def update(self, value: float) -> None: + raise NotImplementedError() + def get_snapshot(self) -> Snapshot: + raise NotImplementedError() + + +class ExpDecayingSample(Sample): """ An exponentially-decaying random sample of longs. Uses Cormode et al's forward-decaying priority reservoir sampling method to produce a @@ -24,7 +54,7 @@ class ExpDecayingSample(object): RESCALE_THREASHOLD = 3600.0 # 1 hour - def __init__(self, size=DEFAULT_SIZE, alpha=DEFAULT_ALPHA, clock=time): + def __init__(self, size: int = DEFAULT_SIZE, alpha: float = DEFAULT_ALPHA, clock: Clock = time) -> None: """ Creates a new L{ExponentiallyDecayingSample}. @@ -44,17 +74,17 @@ def __init__(self, size=DEFAULT_SIZE, alpha=DEFAULT_ALPHA, clock=time): self.alpha = alpha self.clear() - def clear(self): - self.values = {} - self.priorities = [] + def clear(self) -> None: + self.values: dict[float, float] = {} + self.priorities: list[float] = [] self.counter = 0 self.start_time = self.clock.time() self.next_time = self.clock.time() + ExpDecayingSample.RESCALE_THREASHOLD - def get_size(self): + def get_size(self) -> int: return self.counter if self.counter < self.size else self.size - def update(self, value): + def update(self, value: float) -> None: """ Adds a value to the sample. @@ -83,16 +113,16 @@ def update(self, value): else: heapq.heappush(self.priorities, first) - def _rescale_if_necessary(self): + def _rescale_if_necessary(self) -> None: if self.clock.time() >= self.next_time: self._rescale() - def _rescale(self): + def _rescale(self) -> None: self.next_time = self.clock.time() + ExpDecayingSample.RESCALE_THREASHOLD old_start_time = self.start_time self.start_time = self.clock.time() new_values = {} - new_priorities = [] + new_priorities: list[float] = [] for key, val in self.values.items(): priority = key * math.exp(-self.alpha * (self.start_time - old_start_time)) new_values[priority] = val @@ -101,22 +131,21 @@ def _rescale(self): self.priorities = new_priorities self.counter = len(self.values) - def _weight(self, value): + def _weight(self, value: float) -> float: return math.exp(self.alpha * value) - def get_snapshot(self): + def get_snapshot(self) -> Snapshot: return Snapshot(self.values.values()) -class SlidingTimeWindowSample(object): - +class SlidingTimeWindowSample(Sample): """ A sample of measurements made in a sliding time window. """ DEFAULT_WINDOW = 300 - def __init__(self, window=DEFAULT_WINDOW, clock=time): + def __init__(self, window: int = DEFAULT_WINDOW, clock: Clock = time) -> None: """Creates a SlidingTimeWindowSample. :param window: the length of the time window in seconds @@ -127,17 +156,17 @@ def __init__(self, window=DEFAULT_WINDOW, clock=time): self.clock = clock self.clear() - def clear(self): - self.values = [] + def clear(self) -> None: + self.values: list[tuple[float, float]] = [] - def _trim(self): + def _trim(self) -> None: deadline = self.clock.time() - self.window while self.values and self.values[0][0] < deadline: heapq.heappop(self.values) - def update(self, value): + def update(self, value: float) -> None: heapq.heappush(self.values, (self.clock.time(), value)) - def get_snapshot(self): + def get_snapshot(self) -> Snapshot: self._trim() return Snapshot(x[1] for x in self.values) diff --git a/pyformance/stats/snapshot.py b/pyformance/stats/snapshot.py index 67c20e7..9514c13 100644 --- a/pyformance/stats/snapshot.py +++ b/pyformance/stats/snapshot.py @@ -1,8 +1,25 @@ +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + import math +from collections.abc import Iterable class Snapshot(object): - """ This class is used by the histogram meter """ @@ -13,43 +30,43 @@ class Snapshot(object): P99_Q = 0.99 P999_Q = 0.999 - def __init__(self, values): + def __init__(self, values: Iterable[float]) -> None: super(Snapshot, self).__init__() self.values = sorted(values) - def get_size(self): + def get_size(self) -> int: "get current size" return len(self.values) - def get_sum(self): + def get_sum(self) -> float: "get current sum" return float(sum(self.values)) - def get_max(self): + def get_max(self) -> float: "get current maximum value" if not self.values: return 0 return self.values[-1] - def get_min(self): + def get_min(self) -> float: "get current minimum value" if not self.values: return 0 return self.values[0] - def get_mean(self): + def get_mean(self) -> float: "get current mean value" if not self.values: return 0 return float(sum(self.values)) / self.get_size() - def get_stddev(self): + def get_stddev(self) -> float: "get current standard deviation" if not self.values: return 0 return math.sqrt(self.get_var()) - def get_var(self): + def get_var(self) -> float: "get current variance" if not self.values or self.get_size() == 1: return 0 @@ -57,30 +74,30 @@ def get_var(self): square_differences = [(mean - value) ** 2 for value in self.values] return sum(square_differences) / (self.get_size() - 1) - def get_median(self): + def get_median(self) -> float: "get current median" return self.get_percentile(Snapshot.MEDIAN) - def get_75th_percentile(self): + def get_75th_percentile(self) -> float: "get current 75th percentile" return self.get_percentile(Snapshot.P75_Q) - def get_95th_percentile(self): + def get_95th_percentile(self) -> float: "get current 95th percentile" return self.get_percentile(Snapshot.P95_Q) - def get_99th_percentile(self): + def get_99th_percentile(self) -> float: "get current 99th percentile" return self.get_percentile(Snapshot.P99_Q) - def get_999th_percentile(self): + def get_999th_percentile(self) -> float: "get current 999th percentile" return self.get_percentile(Snapshot.P999_Q) - def get_percentile(self, percentile): + def get_percentile(self, percentile: float) -> float: """ get custom percentile - + :param percentile: float value between 0 and 1 """ if percentile < 0 or percentile > 1: diff --git a/setup.py b/setup.py index 35c6094..1b8ef0a 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ import os import functools -import platform from setuptools import setup, find_packages _IN_PACKAGE_DIR = functools.partial(os.path.join, "pyformance") @@ -8,17 +7,12 @@ with open(_IN_PACKAGE_DIR("__version__.py")) as version_file: exec(version_file.read()) -install_requires = ["six"] # optional: ["blinker==1.2"] -if platform.python_version() < "2.7": - install_requires.append("unittest2") - setup( name="pyformance", classifiers=[ "Development Status :: 4 - Beta", "Intended Audience :: Developers", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.11", ], description="Performance metrics, based on Coda Hale's Yammer metrics", license="Apache 2.0", @@ -27,6 +21,6 @@ version=__version__, packages=find_packages(), data_files=[], - install_requires=install_requires, scripts=[], + python_requires=">=3.12", # also update classifiers ) diff --git a/tests/test__carbon_reporter.py b/tests/test__carbon_reporter.py index ddb2cec..cc4386f 100644 --- a/tests/test__carbon_reporter.py +++ b/tests/test__carbon_reporter.py @@ -1,4 +1,5 @@ -from six import BytesIO, PY3 +from io import BytesIO + from pyformance import MetricsRegistry from pyformance.reporters.carbon_reporter import CarbonReporter from tests import TimedTestCase @@ -82,9 +83,7 @@ def test_report_now_plain(self): "hist.min 1 2", "hist.95_percentile 512 2", "hist.75_percentile 160.0 2", - "hist.std_dev 164.94851048466944 2" - if PY3 - else "hist.std_dev 164.948510485 2", + "hist.std_dev 164.94851048466944 2", "hist.max 512 2", "hist.avg 102.3 2", "m1.count 1.0 2", diff --git a/tests/test__counter.py b/tests/test__counter.py index dcf038a..e92bbe2 100644 --- a/tests/test__counter.py +++ b/tests/test__counter.py @@ -1,23 +1,38 @@ +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import pytest from pyformance.meters import Counter -from tests import TimedTestCase -class CounterTestCase(TimedTestCase): - def setUp(self): - super(CounterTestCase, self).setUp() - self.counter = Counter() +@pytest.fixture +def counter(): + return Counter() + - def tearDown(self): - super(CounterTestCase, self).tearDown() +def test__inc(counter): + before = counter.get_count() + counter.inc() + after = counter.get_count() + assert before + 1 == after - def test__inc(self): - before = self.counter.get_count() - self.counter.inc() - after = self.counter.get_count() - self.assertEqual(before + 1, after) - def test__dec(self): - before = self.counter.get_count() - self.counter.dec() - after = self.counter.get_count() - self.assertEqual(before - 1, after) +def test__dec(counter): + before = counter.get_count() + counter.dec() + after = counter.get_count() + assert before - 1 == after diff --git a/tests/test__gauge.py b/tests/test__gauge.py index 4876a83..63553e5 100644 --- a/tests/test__gauge.py +++ b/tests/test__gauge.py @@ -1,19 +1,28 @@ -from pyformance.meters import CallbackGauge -from tests import TimedTestCase +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from pyformance.meters import CallbackGauge -class CallbackGaugeTestCase(TimedTestCase): - def setUp(self): - super(CallbackGaugeTestCase, self).setUp() - self._value = None - self.gauge = CallbackGauge(self._get_val) - def tearDown(self): - super(CallbackGaugeTestCase, self).tearDown() +def test_gauge(): + value = 123 - def _get_val(self): - return self._value + def test_callback() -> int: + return value - def test__value(self): - self._value = 123 - self.assertEqual(self.gauge.get_value(), self._value) + gauge = CallbackGauge(test_callback) + assert gauge.get_value() == 123 diff --git a/tests/test__histogram.py b/tests/test__histogram.py index 24964de..d75964d 100644 --- a/tests/test__histogram.py +++ b/tests/test__histogram.py @@ -1,64 +1,82 @@ -from tests import TimedTestCase +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + from pyformance.meters import Histogram +from pytest import approx + + +def test__a_sample_of_100_from_1000(): + hist = Histogram(100, 0.99) + for i in range(1000): + hist.add(i) + + assert 1000 == hist.get_count() + assert 100 == hist.sample.get_size() + snapshot = hist.get_snapshot() + assert 100 == snapshot.get_size() + + for i in snapshot.values: + assert 0 <= i and i <= 1000 + + assert 999 == hist.get_max() + assert 0 == hist.get_min() + assert 499.5 == hist.get_mean() + assert 83416.6666 == approx(hist.get_var(), 0.0001) + + +def test__a_sample_of_100_from_10(): + hist = Histogram(100, 0.99) + for i in range(10): + hist.add(i) + + assert 10 == hist.get_count() + assert 10 == hist.sample.get_size() + snapshot = hist.get_snapshot() + assert 10 == snapshot.get_size() + + for i in snapshot.values: + assert 0 <= i and i <= 10 + + assert 9 == hist.get_max() + assert 0 == hist.get_min() + assert 4.5 == hist.get_mean() + assert 9.1666 == approx(hist.get_var(), 0.0001) + + +def test__a_long_wait_should_not_corrupt_sample(clock): + hist = Histogram(10, 0.015, clock=clock) + + for i in range(1000): + hist.add(1000 + i) + clock.add(0.1) + + assert hist.get_snapshot().get_size() == 10 + for i in hist.sample.get_snapshot().values: + assert 1000 <= i and i <= 2000 + clock.add(15 * 3600) # 15 hours, should trigger rescale + hist.add(2000) + assert hist.get_snapshot().get_size() == 2 + for i in hist.sample.get_snapshot().values: + assert 1000 <= i and i <= 3000 -class HistogramTestCase(TimedTestCase): - def test__a_sample_of_100_from_1000(self): - hist = Histogram(100, 0.99) - for i in range(1000): - hist.add(i) - - self.assertEqual(1000, hist.get_count()) - self.assertEqual(100, hist.sample.get_size()) - snapshot = hist.get_snapshot() - self.assertEqual(100, snapshot.get_size()) - - for i in snapshot.values: - self.assertTrue(0 <= i and i <= 1000) - - self.assertEqual(999, hist.get_max()) - self.assertEqual(0, hist.get_min()) - self.assertEqual(499.5, hist.get_mean()) - self.assertAlmostEqual(83416.6666, hist.get_var(), delta=0.0001) - - def test__a_sample_of_100_from_10(self): - hist = Histogram(100, 0.99) - for i in range(10): - hist.add(i) - - self.assertEqual(10, hist.get_count()) - self.assertEqual(10, hist.sample.get_size()) - snapshot = hist.get_snapshot() - self.assertEqual(10, snapshot.get_size()) - - for i in snapshot.values: - self.assertTrue(0 <= i and i <= 10) - - self.assertEqual(9, hist.get_max()) - self.assertEqual(0, hist.get_min()) - self.assertEqual(4.5, hist.get_mean()) - self.assertAlmostEqual(9.1666, hist.get_var(), delta=0.0001) - - def test__a_long_wait_should_not_corrupt_sample(self): - hist = Histogram(10, 0.015, clock=self.clock) - - for i in range(1000): - hist.add(1000 + i) - self.clock.add(0.1) - - self.assertEqual(hist.get_snapshot().get_size(), 10) - for i in hist.sample.get_snapshot().values: - self.assertTrue(1000 <= i and i <= 2000) - - self.clock.add(15 * 3600) # 15 hours, should trigger rescale - hist.add(2000) - self.assertEqual(hist.get_snapshot().get_size(), 2) - for i in hist.sample.get_snapshot().values: - self.assertTrue(1000 <= i and i <= 3000) - - for i in range(1000): - hist.add(3000 + i) - self.clock.add(0.1) - self.assertEqual(hist.get_snapshot().get_size(), 10) - for i in hist.sample.get_snapshot().values: - self.assertTrue(3000 <= i and i <= 4000) + for i in range(1000): + hist.add(3000 + i) + clock.add(0.1) + assert hist.get_snapshot().get_size() == 10 + for i in hist.sample.get_snapshot().values: + assert 3000 <= i and i <= 4000 diff --git a/tests/test__meter.py b/tests/test__meter.py index 6b5cdb1..320ef78 100644 --- a/tests/test__meter.py +++ b/tests/test__meter.py @@ -1,80 +1,85 @@ +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + from pyformance.meters import Meter -from tests import TimedTestCase - - -class MeterTestCase(TimedTestCase): - def setUp(self): - super(MeterTestCase, self).setUp() - self.meter = Meter(TimedTestCase.clock) - - def tearDown(self): - super(MeterTestCase, self).tearDown() - - def test__one_minute_rate(self): - self.meter.mark(3) - self.clock.add(5) - self.meter.tick() - - # the EWMA has a rate of 0.6 events/sec after the first tick - self.assertAlmostEqual(0.6, self.meter.get_one_minute_rate(), delta=0.000001) - - self.clock.add(60) - # the EWMA has a rate of 0.22072766 events/sec after 1 minute - self.assertAlmostEqual( - 0.22072766, self.meter.get_one_minute_rate(), delta=0.000001 - ) - - self.clock.add(60) - # the EWMA has a rate of 0.08120117 events/sec after 2 minute - self.assertAlmostEqual( - 0.08120117, self.meter.get_one_minute_rate(), delta=0.000001 - ) - - def test__five_minute_rate(self): - self.meter.mark(3) - self.clock.add(5) - self.meter.tick() - - # the EWMA has a rate of 0.6 events/sec after the first tick - self.assertAlmostEqual(0.6, self.meter.get_five_minute_rate(), delta=0.000001) - - self.clock.add(60) - # the EWMA has a rate of 0.49123845 events/sec after 1 minute - self.assertAlmostEqual( - 0.49123845, self.meter.get_five_minute_rate(), delta=0.000001 - ) - - self.clock.add(60) - # the EWMA has a rate of 0.40219203 events/sec after 2 minute - self.assertAlmostEqual( - 0.40219203, self.meter.get_five_minute_rate(), delta=0.000001 - ) - - def test__fifteen_minute_rate(self): - self.meter.mark(3) - self.clock.add(5) - self.meter.tick() - - # the EWMA has a rate of 0.6 events/sec after the first tick - self.assertAlmostEqual( - 0.6, self.meter.get_fifteen_minute_rate(), delta=0.000001 - ) - - self.clock.add(60) - # the EWMA has a rate of 0.56130419 events/sec after 1 minute - self.assertAlmostEqual( - 0.56130419, self.meter.get_fifteen_minute_rate(), delta=0.000001 - ) - - self.clock.add(60) - # the EWMA has a rate of 0.52510399 events/sec after 2 minute - self.assertAlmostEqual( - 0.52510399, self.meter.get_fifteen_minute_rate(), delta=0.000001 - ) - - def test__mean_rate(self): - self.meter.mark(60) - self.clock.add(60) - self.meter.tick() - val = self.meter.get_mean_rate() - self.assertEqual(1, val) +from pytest import approx + + +def test__one_minute_rate(clock): + meter = Meter(clock) + meter.mark(3) + clock.add(5) + meter.tick() + + # the EWMA has a rate of 0.6 events/sec after the first tick + assert 0.6 == approx(meter.get_one_minute_rate(), 0.000001) + + clock.add(60) + # the EWMA has a rate of 0.22072766 events/sec after 1 minute + assert 0.22072766 == approx(meter.get_one_minute_rate(), 0.000001) + + clock.add(60) + # the EWMA has a rate of 0.08120117 events/sec after 2 minute + assert 0.08120117 == approx(meter.get_one_minute_rate(), 0.000001) + + +def test__five_minute_rate(clock): + meter = Meter(clock) + + meter.mark(3) + clock.add(5) + meter.tick() + + # the EWMA has a rate of 0.6 events/sec after the first tick + assert 0.6 == approx(meter.get_five_minute_rate(), 0.000001) + + clock.add(60) + # the EWMA has a rate of 0.49123845 events/sec after 1 minute + assert 0.49123845 == approx(meter.get_five_minute_rate(), 0.000001) + + clock.add(60) + # the EWMA has a rate of 0.40219203 events/sec after 2 minute + assert 0.40219203 == approx(meter.get_five_minute_rate(), 0.000001) + + +def test__fifteen_minute_rate(clock): + meter = Meter(clock) + + meter.mark(3) + clock.add(5) + meter.tick() + + # the EWMA has a rate of 0.6 events/sec after the first tick + assert 0.6 == approx(meter.get_fifteen_minute_rate(), 0.000001) + + clock.add(60) + # the EWMA has a rate of 0.56130419 events/sec after 1 minute + assert 0.56130419 == approx(meter.get_fifteen_minute_rate(), 0.000001) + + clock.add(60) + # the EWMA has a rate of 0.52510399 events/sec after 2 minute + assert 0.52510399 == approx(meter.get_fifteen_minute_rate(), 0.000001) + + +def test__mean_rate(clock): + meter = Meter(clock) + + meter.mark(60) + clock.add(60) + meter.tick() + val = meter.get_mean_rate() + assert 1 == val diff --git a/tests/test__moving_average.py b/tests/test__moving_average.py index 5a547e9..3490dae 100644 --- a/tests/test__moving_average.py +++ b/tests/test__moving_average.py @@ -1,135 +1,156 @@ +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + from pyformance.stats.moving_average import ExpWeightedMovingAvg -from tests import TimedTestCase - - -class EWMATests(TimedTestCase): - def test_one_minute_EWMA_five_sec_tick(self): - self.ewma = ExpWeightedMovingAvg(1, clock=self.clock) - - self.ewma.add(3) - self.clock.add(5) - self.ewma.tick() - - for expected_rate in [ - 0.6, - 0.22072766, - 0.08120117, - 0.02987224, - 0.01098938, - 0.00404277, - 0.00148725, - 0.00054713, - 0.00020128, - 0.00007405, - ]: - self.assertAlmostEqual(self.ewma.get_rate(), expected_rate) - self.clock.add(60) - - def test_five_minute_EWMA_five_sec_tick(self): - self.ewma = ExpWeightedMovingAvg(5, clock=self.clock) - - self.ewma.add(3) - self.clock.add(5) - self.ewma.tick() - - for expected_rate in [ - 0.6, - 0.49123845, - 0.40219203, - 0.32928698, - 0.26959738, - 0.22072766, - 0.18071653, - 0.14795818, - 0.12113791, - 0.09917933, - ]: - self.assertAlmostEqual(self.ewma.get_rate(), expected_rate) - self.clock.add(60) - - def test_fifteen_minute_EWMA_five_sec_tick(self): - self.ewma = ExpWeightedMovingAvg(15, clock=self.clock) - - self.ewma.add(3) - self.clock.add(5) - self.ewma.tick() - - for expected_rate in [ - 0.6, - 0.56130419, - 0.52510399, - 0.49123845, - 0.45955700, - 0.42991879, - 0.40219203, - 0.37625345, - 0.35198773, - 0.32928698, - ]: - self.assertAlmostEqual(self.ewma.get_rate(), expected_rate) - self.clock.add(60) - - def test_one_minute_EWMA_one_minute_tick(self): - self.ewma = ExpWeightedMovingAvg(1, 60, clock=self.clock) - self.ewma.add(3) - self.clock.add(5) - self.ewma.tick() - - for expected_rate in [ - 0.6, - 0.22072766, - 0.08120117, - 0.02987224, - 0.01098938, - 0.00404277, - 0.00148725, - 0.00054713, - 0.00020128, - 0.00007405, - ]: - self.assertAlmostEqual(self.ewma.get_rate(), expected_rate) - self.clock.add(60) - - def test_five_minute_EWMA_one_minute_tick(self): - self.ewma = ExpWeightedMovingAvg(5, 60, clock=self.clock) - - self.ewma.add(3) - self.clock.add(5) - self.ewma.tick() - - for expected_rate in [ - 0.6, - 0.49123845, - 0.40219203, - 0.32928698, - 0.26959738, - 0.22072766, - 0.18071653, - 0.14795818, - 0.12113791, - 0.09917933, - ]: - self.assertAlmostEqual(self.ewma.get_rate(), expected_rate) - self.clock.add(60) - - def test_fifteen_minute_EWMA_one_minute_tick(self): - self.ewma = ExpWeightedMovingAvg(15, 60, clock=self.clock) - - self.ewma.add(3) - self.clock.add(5) - self.ewma.tick() - - for expected_rate in [ - 0.6, - 0.56130419, - 0.52510399, - 0.49123845, - 0.45955700, - 0.42991879, - 0.40219203, - 0.37625345, - 0.35198773, - 0.32928698, - ]: - self.assertAlmostEqual(self.ewma.get_rate(), expected_rate) - self.clock.add(60) +from pytest import approx + + +def test_one_minute_EWMA_five_sec_tick(clock): + ewma = ExpWeightedMovingAvg(1, clock=clock) + + ewma.add(3) + clock.add(5) + ewma.tick() + + for expected_rate in [ + 0.6, + 0.22072766, + 0.08120117, + 0.02987224, + 0.01098938, + 0.00404277, + 0.00148725, + 0.00054713, + 0.00020128, + 0.00007405, + ]: + assert ewma.get_rate() == approx(expected_rate, 0.0001) + clock.add(60) + + +def test_five_minute_EWMA_five_sec_tick(clock): + ewma = ExpWeightedMovingAvg(5, clock=clock) + + ewma.add(3) + clock.add(5) + ewma.tick() + + for expected_rate in [ + 0.6, + 0.49123845, + 0.40219203, + 0.32928698, + 0.26959738, + 0.22072766, + 0.18071653, + 0.14795818, + 0.12113791, + 0.09917933, + ]: + assert ewma.get_rate() == approx(expected_rate) + clock.add(60) + + +def test_fifteen_minute_EWMA_five_sec_tick(clock): + ewma = ExpWeightedMovingAvg(15, clock=clock) + + ewma.add(3) + clock.add(5) + ewma.tick() + + for expected_rate in [ + 0.6, + 0.56130419, + 0.52510399, + 0.49123845, + 0.45955700, + 0.42991879, + 0.40219203, + 0.37625345, + 0.35198773, + 0.32928698, + ]: + assert ewma.get_rate() == approx(expected_rate) + clock.add(60) + + +def test_one_minute_EWMA_one_minute_tick(clock): + ewma = ExpWeightedMovingAvg(1, 60, clock=clock) + ewma.add(3) + clock.add(5) + ewma.tick() + + for expected_rate in [ + 0.6, + 0.22072766, + 0.08120117, + 0.02987224, + 0.01098938, + 0.00404277, + 0.00148725, + 0.00054713, + 0.00020128, + 0.00007405, + ]: + assert ewma.get_rate() == approx(expected_rate, 0.0001) + clock.add(60) + + +def test_five_minute_EWMA_one_minute_tick(clock): + ewma = ExpWeightedMovingAvg(5, 60, clock=clock) + + ewma.add(3) + clock.add(5) + ewma.tick() + + for expected_rate in [ + 0.6, + 0.49123845, + 0.40219203, + 0.32928698, + 0.26959738, + 0.22072766, + 0.18071653, + 0.14795818, + 0.12113791, + 0.09917933, + ]: + assert ewma.get_rate() == approx(expected_rate) + clock.add(60) + + +def test_fifteen_minute_EWMA_one_minute_tick(clock): + ewma = ExpWeightedMovingAvg(15, 60, clock=clock) + + ewma.add(3) + clock.add(5) + ewma.tick() + + for expected_rate in [ + 0.6, + 0.56130419, + 0.52510399, + 0.49123845, + 0.45955700, + 0.42991879, + 0.40219203, + 0.37625345, + 0.35198773, + 0.32928698, + ]: + assert ewma.get_rate() == approx(expected_rate) + clock.add(60) diff --git a/tests/test__timer.py b/tests/test__timer.py index 147604f..3fbbbd0 100644 --- a/tests/test__timer.py +++ b/tests/test__timer.py @@ -1,22 +1,61 @@ +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + from pyformance.meters import Timer -from tests import TimedTestCase -class TimerTestCase(TimedTestCase): - def setUp(self): - super(TimerTestCase, self).setUp() - self.timer = Timer() +def test__start_stop_clear(clock): + timer = Timer(clock=clock) + + context = timer.time() + clock.add(1) + context.stop() + + assert timer.get_count() == 1 + assert timer.get_max() == 1 + assert timer.get_min() == 1 + assert timer.get_mean() == 1 + assert timer.get_sum() == 1 + assert timer.get_mean_rate() == 1 + + context = timer.time() + clock.add(2) + context.stop() - def tearDown(self): - super(TimerTestCase, self).tearDown() + assert timer.get_count() == 2 + assert timer.get_max() == 2 + assert timer.get_min() == 1 + assert timer.get_mean() == 1.5 + assert timer.get_snapshot().get_median() == 1.5 + assert timer.get_sum() == 3 + assert timer.get_mean_rate() == 2.0 / 3 - def test__start_stop_clear(self): - context = self.timer.time() - self.clock.add(1) - context.stop() + context = timer.time() + clock.add(1) + context.stop() - self.assertEqual(self.timer.get_count(), 1) + assert timer.get_count() == 3 + assert timer.get_max() == 2 + assert timer.get_min() == 1 + assert timer.get_mean() == 4.0 / 3 + assert timer.get_snapshot().get_median() == 1 + assert timer.get_sum() == 4 + assert timer.get_mean_rate() == 0.75 - self.timer.clear() + timer.clear() - self.assertEqual(self.timer.get_count(), 0) + assert timer.get_count() == 0 diff --git a/tox.ini b/tox.ini index e6a7bd9..93fdaeb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34,py35,py36 +envlist = py312 [testenv] commands= From ee0f7b6851a94841fcfb067037c1f61b8c743aa6 Mon Sep 17 00:00:00 2001 From: wouter Date: Mon, 23 Jun 2025 11:20:46 +0200 Subject: [PATCH 2/2] Add new files --- MANIFEST.in | 1 + pyformance/py.typed | 0 tests/conftest.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 pyformance/py.typed create mode 100644 tests/conftest.py diff --git a/MANIFEST.in b/MANIFEST.in index c1a7121..651e44d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include LICENSE include README.md +include pyformance/py.typed \ No newline at end of file diff --git a/pyformance/py.typed b/pyformance/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a6e655d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,35 @@ +""" +Copyright 2014 Omer Gertel +Copyright 2025 Inmanta + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import pytest + + +class ManualClock(object): + def __init__(self): + super(ManualClock, self).__init__() + self.now = 0 + + def add(self, value): + self.now = self.now + value + + def time(self): + return self.now + + +@pytest.fixture +def clock(): + return ManualClock()