From a939e6503c2106003a65aae59138993429fa6c4e Mon Sep 17 00:00:00 2001 From: Bastian Venthur Date: Sat, 6 Aug 2022 21:56:41 +0200 Subject: [PATCH 1/6] rebuild only files that need re-building we compare the mtimes of src and dst --- blag/blag.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/blag/blag.py b/blag/blag.py index e84bc2b..c47fc2b 100644 --- a/blag/blag.py +++ b/blag/blag.py @@ -293,6 +293,18 @@ def process_markdown(convertibles, input_dir, output_dir, for src, dst in convertibles: logger.debug(f'Processing {src}') + # see first if the dst actually needs re-building. for that we compare + # the mtimes and assume mtime_dst > mtime_src means that it needs not + # to be rebuilt + if os.path.exists(f'{output_dir}/{dst}'): + mtime_src = os.stat(f'{input_dir}/{src}').st_mtime + mtime_dst = os.stat(f'{output_dir}/{dst}').st_mtime + if mtime_dst >= mtime_src: + logger.debug( + 'Skipping, as target exists and is newer than source.' + ) + continue + with open(f'{input_dir}/{src}', 'r') as fh: body = fh.read() From 5943dab6907b4b49c48c2962c610bbcace3d324f Mon Sep 17 00:00:00 2001 From: Bastian Venthur Date: Mon, 1 Jul 2024 13:47:57 +0200 Subject: [PATCH 2/6] wip --- tests/test_markdown.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_markdown.py b/tests/test_markdown.py index 5e393d2..310da89 100644 --- a/tests/test_markdown.py +++ b/tests/test_markdown.py @@ -1,11 +1,14 @@ """Test markdown module.""" from datetime import datetime +import os from typing import Any import markdown import pytest +import blag +from blag.blag import build from blag.markdown import convert_markdown, markdown_factory @@ -111,3 +114,28 @@ def test_smarty_code() -> None: assert "mdash" not in html assert "ndash" not in html assert "hellip" not in html + + +def test_performance(args) -> None: + # create 1000 random markdown files in the content directory + with open(os.path.join(blag.__path__[0], "content", "testpage.md")) as fh: + markdown = fh.read() + for i in range(1000): + with open(f"content/{i}.md", "w") as f: + f.write(markdown) + + from time import time + + t = time() + build(args) + print(time() - t) + + import cProfile + + t = time() + cProfile.run("build(args)") + #build(args) + print(time() - t) + + + 1 / 0 From c4f3c09730be2442c62a15bbe007b38a39c80dd2 Mon Sep 17 00:00:00 2001 From: Bastian Venthur Date: Mon, 8 Jul 2024 15:35:58 +0200 Subject: [PATCH 3/6] WIP --- Makefile | 4 ++++ tests/test_markdown.py | 28 ---------------------------- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index 5b8208e..2765cb4 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,10 @@ serve-docs: $(VENV) manpage: $(VENV) help2man $(BIN)/blag --no-info -n "blog-aware, static site generator" -o debian/blag.1 +.PHONY: benchmark +benchmark: $(VENV) + $(BIN)/pytest --no-cov -rP tests/benchmark.py + .PHONY: clean clean: rm -rf build dist *.egg-info diff --git a/tests/test_markdown.py b/tests/test_markdown.py index 310da89..5e393d2 100644 --- a/tests/test_markdown.py +++ b/tests/test_markdown.py @@ -1,14 +1,11 @@ """Test markdown module.""" from datetime import datetime -import os from typing import Any import markdown import pytest -import blag -from blag.blag import build from blag.markdown import convert_markdown, markdown_factory @@ -114,28 +111,3 @@ def test_smarty_code() -> None: assert "mdash" not in html assert "ndash" not in html assert "hellip" not in html - - -def test_performance(args) -> None: - # create 1000 random markdown files in the content directory - with open(os.path.join(blag.__path__[0], "content", "testpage.md")) as fh: - markdown = fh.read() - for i in range(1000): - with open(f"content/{i}.md", "w") as f: - f.write(markdown) - - from time import time - - t = time() - build(args) - print(time() - t) - - import cProfile - - t = time() - cProfile.run("build(args)") - #build(args) - print(time() - t) - - - 1 / 0 From a3da95ea19387b77f0fe9462b11984fd8c7443eb Mon Sep 17 00:00:00 2001 From: Bastian Venthur Date: Tue, 29 Oct 2024 12:13:05 +0100 Subject: [PATCH 4/6] added benchmark --- tests/benchmark.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/benchmark.py diff --git a/tests/benchmark.py b/tests/benchmark.py new file mode 100644 index 0000000..63a5aa2 --- /dev/null +++ b/tests/benchmark.py @@ -0,0 +1,30 @@ +import os + +import blag +from blag.blag import build + + +def test_performance(args) -> None: + # create 1000 random markdown files in the content directory + with open(os.path.join(blag.__path__[0], "content", "testpage.md")) as fh: + markdown = fh.read() + for i in range(10000): + with open(f"content/{i}.md", "w") as f: + f.write(markdown) + f.write(str(i)) + + from time import time + + t = time() + build(args) + print(time() - t) + + import cProfile + + t = time() + #cProfile.run("build(args)") + build(args) + print(time() - t) + + + 1 / 0 From c94f9793b07fac5f88add6c4226ad1fa1b80b642 Mon Sep 17 00:00:00 2001 From: Bastian Venthur Date: Wed, 22 Jan 2025 14:46:14 +0100 Subject: [PATCH 5/6] improved performance test --- Makefile | 2 +- tests/benchmark.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 2765cb4..cbaadaa 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ manpage: $(VENV) .PHONY: benchmark benchmark: $(VENV) - $(BIN)/pytest --no-cov -rP tests/benchmark.py + $(BIN)/pytest --no-cov -s -rP tests/benchmark.py .PHONY: clean clean: diff --git a/tests/benchmark.py b/tests/benchmark.py index 63a5aa2..3766f21 100644 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -5,10 +5,12 @@ def test_performance(args) -> None: - # create 1000 random markdown files in the content directory + FILES = 1000 + print(f"Generating {FILES} files") + # create random markdown files in the content directory with open(os.path.join(blag.__path__[0], "content", "testpage.md")) as fh: markdown = fh.read() - for i in range(10000): + for i in range(FILES): with open(f"content/{i}.md", "w") as f: f.write(markdown) f.write(str(i)) @@ -17,14 +19,13 @@ def test_performance(args) -> None: t = time() build(args) - print(time() - t) - - import cProfile + t_first = time() - t + print(t_first) t = time() - #cProfile.run("build(args)") build(args) - print(time() - t) - + t_second = time() - t + print(t_second) + print(f"First run: {t_first:.2f}s, second run: {t_second:.2f}s") + print(f"Speedup: {t_first/t_second:.2f}") - 1 / 0 From 60bd86ec225ffff8f1a59778a6f285982e355bda Mon Sep 17 00:00:00 2001 From: Bastian Venthur Date: Wed, 22 Jan 2025 15:17:28 +0100 Subject: [PATCH 6/6] finalized benchmark --- Makefile | 2 +- tests/benchmark.py | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index cbaadaa..a46692b 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ manpage: $(VENV) .PHONY: benchmark benchmark: $(VENV) - $(BIN)/pytest --no-cov -s -rP tests/benchmark.py + $(BIN)/pytest --no-cov -capture=no -rA tests/benchmark.py .PHONY: clean clean: diff --git a/tests/benchmark.py b/tests/benchmark.py index 3766f21..9dbc309 100644 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -1,15 +1,24 @@ +"""Benchmark the performance of the blag build command.""" + +import logging import os +from argparse import Namespace + +from pytest import LogCaptureFixture import blag from blag.blag import build -def test_performance(args) -> None: - FILES = 1000 - print(f"Generating {FILES} files") +def test_performance(args: Namespace, caplog: LogCaptureFixture) -> None: + """Test performance of the build command.""" + caplog.set_level(logging.ERROR) + + FILES = 10000 + print(f"Generating {FILES} markdown files") # create random markdown files in the content directory with open(os.path.join(blag.__path__[0], "content", "testpage.md")) as fh: - markdown = fh.read() + markdown = fh.read() for i in range(FILES): with open(f"content/{i}.md", "w") as f: f.write(markdown) @@ -20,12 +29,9 @@ def test_performance(args) -> None: t = time() build(args) t_first = time() - t - print(t_first) t = time() build(args) t_second = time() - t - print(t_second) print(f"First run: {t_first:.2f}s, second run: {t_second:.2f}s") print(f"Speedup: {t_first/t_second:.2f}") -