diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..2f40c237 --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 99 +exclude = .tox, .git, __pycache__, .ipynb_checkpoints diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7d1cb865..5803fd9d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -59,7 +59,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install package and dependencies - run: pip install invoke rundoc . + run: pip install invoke rundoc tomli . - name: invoke readme run: invoke readme @@ -119,6 +119,6 @@ jobs: name: Install dependencies - Windows with Python 3.6 run: python -m pip install pywinpty==2.0.1 - name: Install package and dependencies - run: pip install invoke jupyter matplotlib . + run: pip install invoke jupyter matplotlib tomli . - name: invoke tutorials run: invoke tutorials diff --git a/Makefile b/Makefile index 44ef6a0e..a6ca5780 100644 --- a/Makefile +++ b/Makefile @@ -81,11 +81,9 @@ install-test: clean-build clean-pyc ## install the package and test dependencies install-develop: clean-build clean-pyc ## install the package in editable mode and dependencies for development pip install -e .[dev] -MINIMUM := $(shell sed -n '/install_requires = \[/,/]/p' setup.py | grep -v -e '[][]' | sed 's/ *\(.*\),$?$$/\1/g' | tr '>' '=') - .PHONY: install-minimum install-minimum: ## install the minimum supported versions of the package dependencies - pip install $(MINIMUM) + invoke install_minimum .PHONY: check-dependencies check-dependencies: ## test if there are any broken dependencies @@ -164,8 +162,7 @@ serve-docs: view-docs ## compile the docs watching for changes .PHONY: dist dist: clean ## builds source and wheel package - python setup.py sdist - python setup.py bdist_wheel + python -m build --wheel --sdist ls -l dist .PHONY: publish-confirm diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..6ec142d4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,193 @@ +# Project Metadata + +[project] +name = 'orion' +description='Orion is a machine learning library built for unsupervised time series anomaly detection.' +authors = [{ name = 'MIT Data To AI Lab', email = 'dailabmit@gmail.com' }] +classifiers = [ + 'Development Status :: 2 - Pre-Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Natural Language :: English', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', +] +keywords = ['orion', 'anomaly-detection', 'timeseries'] +version='0.6.1.dev0' +license = { text = 'MIT license' } +requires-python = '>=3.8,<3.12' +readme = 'README.md' +dependencies = [ + 'tensorflow>=2.2,<2.15', + 'numpy>=1.17.5,<2', + 'pandas>=1,<2', + 'numba>=0.48,<0.60', + 's3fs>=0.2.2,<0.5', + 'mlblocks>=0.6.1,<0.7', + 'ml-stars>=0.2,<0.3', + 'scikit-learn>=0.22.1,<1.2', + 'tabulate>=0.8.3,<0.9', + 'pyts>=0.11,<0.14', + 'torch>=1.4', + 'azure-cognitiveservices-anomalydetector>=0.3,<0.4', + 'xlsxwriter>=1.3.6,<1.4', + 'tqdm>=4.36.1', + 'stumpy>=1.7,<1.11', + 'ncps', + + # fix conflict + 'protobuf<4', + + # fails on python 3.6 + 'opencv-python<4.7', +] + +[project.optional-dependencies] +test = [ + 'pytest>=3.4.2', + 'pytest-cov>=2.6.0', + 'rundoc>=0.4.3,<0.5', + 'pytest-runner>=2.11.1', + 'tomli>=2.0.0,<3', +] +dev = [ + 'orion[test]', + + # general + 'pip>=9.0.1', + 'bumpversion>=0.5.3,<0.6', + 'watchdog>=0.8.3,<0.11', + + # docs + 'docutils>=0.12,<0.18', + 'm2r2>=0.2.5,<0.3', + 'nbsphinx>=0.5.0,<0.7', + 'Sphinx>=3,<3.3', + 'pydata-sphinx-theme<0.5', + 'markupsafe<2.1.0', + 'ipython>=6.5,<9', + 'Jinja2>=2,<3', + + # fails on Sphinx < v3.4 + 'alabaster<=0.7.12', + # fails on Sphins < v5.0 + 'sphinxcontrib-applehelp<1.0.8', + 'sphinxcontrib-devhelp<1.0.6', + 'sphinxcontrib-htmlhelp<2.0.5', + 'sphinxcontrib-serializinghtml<1.1.10', + 'sphinxcontrib-qthelp<1.0.7', + + # style check + 'flake8>=3.7.7,<4', + 'isort>=4.3.4,<5', + + # fix style issues + 'autoflake>=1.2,<2', + 'autopep8>=1.4.3,<2', + 'importlib-metadata<5', + + # distribute on PyPI + 'twine>=1.10.0,<4', + 'wheel>=0.30.0', + + # Advanced testing + 'coverage>=4.5.1,<6', + 'tox>=2.9.1,<4', + 'invoke', +] + +[project.urls] +"Source Code"= "https://github.com/sintel-dev/Orion/" +"Issue Tracker" = "https://github.com/sintel-dev/Orion/issues" +"Changes" = "https://github.com/sintel-dev/Orion/blob/main/HISTORY.md" +"Chat" = "https://join.slack.com/t/sintel-space/shared_invite/zt-q147oimb-4HcphcxPfDAM0O9_4PaUtw" + +[project.entry-points] +orion = { main = 'orion.__main__:main' } +mlblocks = { primitives = 'orion:MLBLOCKS_PRIMITIVES', pipelines = 'orion:MLBLOCKS_PIPELINES' } + +[tool.setuptools] +include-package-data = true +license-files = ['LICENSE'] + +[tool.setuptools.packages.find] +include = ['orion', 'orion.*'] +namespaces = false + +[tool.setuptools.package-data] +'*' = [ + 'AUTHORS.rst', + 'CONTRIBUTING.rst', + 'HISTORY.md', + 'README.md', + '*.md', + '*.rst', + 'conf.py', + 'Makefile', + 'make.bat', + '*.jpg', + '*.png', + '*.gif' +] +'tests' = ['*'] + +[tool.setuptools.exclude-package-data] +'*' = [ + '* __pycache__', + '*.py[co]', +] + +[tool.setuptools.dynamic] +version = {attr = 'orion.__version__'} + +[tool.isort] +line_length = 99 +lines_between_types = 0 +multi_line_output = 4 +not_skip = ['__init__.py'] +use_parentheses = true +include_trailing_comment = true + +[tool.pytest.ini_options] +collect_ignore = ['pyproject.toml'] + +[tool.bumpversion] +current_version = '0.6.1.dev0' +parse = '(?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))?' +serialize = [ + '{major}.{minor}.{patch}.{release}{candidate}', + '{major}.{minor}.{patch}' +] +search = '{current_version}' +replace = '{new_version}' +regex = false +ignore_missing_version = false +tag = true +sign_tags = false +tag_name = 'v{new_version}' +tag_message = 'Bump version: {current_version} → {new_version}' +allow_dirty = false +commit = true +message = 'Bump version: {current_version} → {new_version}' +commit_args = '' + +[tool.bumpversion.parts.release] +first_value = 'dev' +optional_value = 'release' +values = [ + 'dev', + 'release' +] + +[tool.bumpversion.files] +filename = "orion/__init__.py" +search = "__version__ = '{current_version}'" +replace = "__version__ = '{new_version}'" + +[build-system] +requires = ['setuptools', 'wheel'] +build-backend = 'setuptools.build_meta' + diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index e877accb..00000000 --- a/setup.cfg +++ /dev/null @@ -1,47 +0,0 @@ -[bumpversion] -current_version = 0.6.1.dev0 -commit = True -tag = True -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? -serialize = - {major}.{minor}.{patch}.{release}{candidate} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = release -first_value = dev -values = - dev - release - -[bumpversion:part:candidate] - -[bumpversion:file:setup.py] -search = version='{current_version}' -replace = version='{new_version}' - -[bumpversion:file:orion/__init__.py] -search = __version__ = '{current_version}' -replace = __version__ = '{new_version}' - -[bdist_wheel] -universal = 1 - -[flake8] -max-line-length = 99 -exclude = .tox, .git, __pycache__, .ipynb_checkpoints - -[isort] -include_trailing_comment = True -line_length = 99 -lines_between_types = 0 -multi_line_output = 4 -not_skip = __init__.py -use_parentheses = True - -[aliases] -test = pytest - -[tool:pytest] -collect_ignore = ['setup.py'] - diff --git a/setup.py b/setup.py deleted file mode 100644 index 1d8462a7..00000000 --- a/setup.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from setuptools import setup, find_packages - -try: - with open('README.md', encoding='utf-8') as readme_file: - readme = readme_file.read() -except IOError: - readme = '' - -try: - with open('HISTORY.md', encoding='utf-8') as history_file: - history = history_file.read() -except IOError: - history = '' - - -install_requires = [ - 'tensorflow>=2.2,<2.15', - 'numpy>=1.17.5,<2', - 'pandas>=1,<2', - 'numba>=0.48,<0.60', - 's3fs>=0.2.2,<0.5', - 'mlblocks>=0.6.1,<0.7', - 'ml-stars>=0.2,<0.3', - 'scikit-learn>=0.22.1,<1.2', - 'tabulate>=0.8.3,<0.9', - 'pyts>=0.11,<0.14', - 'torch>=1.4', - 'azure-cognitiveservices-anomalydetector>=0.3,<0.4', - 'xlsxwriter>=1.3.6,<1.4', - 'tqdm>=4.36.1', - 'stumpy>=1.7,<1.11', - 'ncps', - - # fix conflict - 'protobuf<4', - - # fails on python 3.6 - 'opencv-python<4.7', -] - -setup_requires = [ - 'pytest-runner>=2.11.1', -] - -tests_require = [ - 'pytest>=3.4.2', - 'pytest-cov>=2.6.0', - 'rundoc>=0.4.3,<0.5', -] - -development_requires = [ - # general - 'pip>=9.0.1', - 'bumpversion>=0.5.3,<0.6', - 'watchdog>=0.8.3,<0.11', - - # docs - 'docutils>=0.12,<0.18', - 'm2r2>=0.2.5,<0.3', - 'nbsphinx>=0.5.0,<0.7', - 'Sphinx>=3,<3.3', - 'pydata-sphinx-theme<0.5', - 'markupsafe<2.1.0', - 'ipython>=6.5,<9', - 'Jinja2>=2,<3', - - # fails on Sphinx < v3.4 - 'alabaster<=0.7.12', - # fails on Sphins < v5.0 - 'sphinxcontrib-applehelp<1.0.8', - 'sphinxcontrib-devhelp<1.0.6', - 'sphinxcontrib-htmlhelp<2.0.5', - 'sphinxcontrib-serializinghtml<1.1.10', - 'sphinxcontrib-qthelp<1.0.7', - - # style check - 'flake8>=3.7.7,<4', - 'isort>=4.3.4,<5', - - # fix style issues - 'autoflake>=1.2,<2', - 'autopep8>=1.4.3,<2', - 'importlib-metadata<5', - - # distribute on PyPI - 'twine>=1.10.0,<4', - 'wheel>=0.30.0', - - # Advanced testing - 'coverage>=4.5.1,<6', - 'tox>=2.9.1,<4', - 'invoke', -] - -setup( - author="MIT Data To AI Lab", - author_email='dailabmit@gmail.com', - classifiers=[ - 'Development Status :: 2 - Pre-Alpha', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Natural Language :: English', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - ], - description="Orion is a machine learning library built for unsupervised time series anomaly detection.", - entry_points={ - 'console_scripts': [ - 'orion=orion.__main__:main' - ], - 'mlblocks': [ - 'primitives=orion:MLBLOCKS_PRIMITIVES', - 'pipelines=orion:MLBLOCKS_PIPELINES' - ], - }, - extras_require={ - 'test': tests_require, - 'dev': development_requires + tests_require, - }, - include_package_data=True, - install_requires=install_requires, - keywords='orion', - license="MIT license", - long_description=readme + '\n\n' + history, - long_description_content_type='text/markdown', - name='orion-ml', - packages=find_packages(include=['orion', 'orion.*']), - python_requires='>=3.8,<3.12', - setup_requires=setup_requires, - test_suite='tests', - tests_require=tests_require, - url='https://github.com/sintel-dev/Orion', - version='0.6.1.dev0', - zip_safe=False, -) diff --git a/tasks.py b/tasks.py index b3019427..e8012cc5 100644 --- a/tasks.py +++ b/tasks.py @@ -1,69 +1,60 @@ import glob -import operator import os -import platform -import re import shutil import stat +import sys from pathlib import Path +import tomli from invoke import task +from packaging.requirements import Requirement +from packaging.version import Version -COMPARISONS = { - '>=': operator.ge, - '>': operator.gt, - '<': operator.lt, - '<=': operator.le -} +def _get_package_names(dependencies): + packages = list(map(lambda dep: Requirement(dep).name, dependencies)) + return packages -@task -def pytest(c): - c.run('python -m pytest --cov=orion') +def _get_minimum_versions(dependencies, python_version): + min_versions = {} + for dependency in dependencies: + if '@' in dependency: + name, url = dependency.split(' @ ') + min_versions[name] = f'{name} @ {url}' + continue + + req = Requirement(dependency) + if ';' in dependency: + marker = req.marker + if marker and not marker.evaluate({'python_version': python_version}): + continue # python version does not match + + if req.name not in min_versions: + min_version = next((spec.version for spec in req.specifier if spec.operator in ('>=', '==')), None) + if min_version: + min_versions[req.name] = f'{req.name}=={min_version}' + + elif '@' not in min_versions[req.name]: + existing_version = Version(min_versions[req.name].split('==')[1]) + new_version = next((spec.version for spec in req.specifier if spec.operator in ('>=', '==')), existing_version) + if new_version > existing_version: + min_versions[req.name] = f'{req.name}=={new_version}' + + return list(min_versions.values()) @task def install_minimum(c): - with open('setup.py', 'r') as setup_py: - lines = setup_py.read().splitlines() - - versions = [] - started = False - for line in lines: - if started: - if line == ']': - started = False - continue - - line = line.strip() - if line.startswith('#'): # ignore comment - continue - - # get specific package version based on declared python version - if 'python_version' in line: - python_version = re.search(r"python_version(<=?|>=?)\'(\d\.?)+\'", line) - operation = python_version.group(1) - version_number = python_version.group(0).split(operation)[-1].replace("'", "") - if COMPARISONS[operation](platform.python_version(), version_number): - line = line.split(";")[0] - else: - continue - - line = re.sub(r"""['",]""", '', line) - line = re.sub(r'>=?', '==', line) - if '==' in line: - line = re.sub(r',?<=?[\d.]*,?', '', line) - - elif re.search(r',?<=?[\d.]*,?', line): - line = f"'{line}'" - - versions.append(line) - - elif line.startswith('install_requires = ['): - started = True - - c.run(f'python -m pip install {" ".join(versions)}') + with open('pyproject.toml', 'rb') as pyproject_file: + pyproject_data = tomli.load(pyproject_file) + + dependencies = pyproject_data.get('project', {}).get('dependencies', []) + python_version = '.'.join(map(str, sys.version_info[:2])) + minimum_versions = _get_minimum_versions(dependencies, python_version) + + if minimum_versions: + c.run(f'python -m pip install {" ".join(minimum_versions)}') @task @@ -73,6 +64,11 @@ def minimum(c): c.run('python -m pytest') +@task +def pytest(c): + c.run('python -m pytest --cov=orion') + + @task def readme(c): test_path = Path('tests/readme_test') @@ -110,27 +106,11 @@ def lint(c): @task def checkdeps(c, path): - with open('setup.py', 'r') as setup_py: - lines = setup_py.read().splitlines() - - packages = [] - started = False - for line in lines: - if started: - if line == ']': - started = False - continue - - line = line.strip() - if line.startswith('#') or not line: # ignore comment - continue - - line = re.split(r'>?=?=1.20.0,<2;python_version<'3.10'", + "numpy>=1.23.3,<2;python_version>='3.10'", + "pandas>=1.2.0,<2;python_version<'3.10'", + "pandas>=1.3.0,<2;python_version>='3.10'", + 'tensorflow>=2.2,<2.15', + 'pandas @ git+https://github.com/pandas-dev/pandas.git@master#egg=pandas' + ] + + +def test_get_package_names(dependencies): + # Run + packages = _get_package_names(dependencies) + + # Assert + expected = [ + 'numpy', + 'numpy', + 'pandas', + 'pandas', + 'tensorflow', + 'pandas' + ] + + assert packages == expected + + +def test_get_minimum_versions(dependencies): + # Run + minimum_versions_39 = _get_minimum_versions(dependencies, '3.9') + minimum_versions_310 = _get_minimum_versions(dependencies, '3.10') + + # Assert + expected_versions_39 = [ + 'numpy==1.20.0', + 'pandas @ git+https://github.com/pandas-dev/pandas.git@master#egg=pandas', + 'tensorflow==2.2', + ] + expected_versions_310 = [ + 'numpy==1.23.3', + 'pandas @ git+https://github.com/pandas-dev/pandas.git@master#egg=pandas', + 'tensorflow==2.2', + ] + + assert minimum_versions_39 == expected_versions_39 + assert minimum_versions_310 == expected_versions_310