diff --git a/requirements.txt b/requirements.txt index 7bc5eb9..61b4099 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ elasticsearch==5.0.1 flask-helpers==0.1 gunicorn==19.6.0 jsonschema>=2.0.0,!=2.5.0,<3.0.0 # MIT +six==1.10.0 diff --git a/runbook/api/utils.py b/runbook/api/utils.py new file mode 100644 index 0000000..555a80d --- /dev/null +++ b/runbook/api/utils.py @@ -0,0 +1,79 @@ +# Copyright 2016: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import functools + +import flask +import six + +from runbook import config + + +def check_regions(api_type): + def check_decorator(func): + regions = set(config.get_config(api_type)["regions"]) + + @functools.wraps(func) + def checker(region=None, *args, **kwargs): + + if region is not None and region not in regions: + return flask.jsonify( + {"error": "Region {} Not Found".format(region)}), 404 + return func(region, *args, **kwargs) + + return checker + return check_decorator + + +def convert_run(hit): + body = {k: v for k, v in hit["_source"].items()} + + if "_index" in hit: + body["region_id"] = hit["_index"][len("ms_runbooks_"):] + + runbook = None + if 'inner_hits' in hit: + inner_hit = hit['inner_hits']['parent']['hits']['hits'] + if inner_hit: + runbook = convert_runbook(inner_hit[0]) + + body["runbook"] = runbook + + body["id"] = hit["_id"] + return body + + +def convert_runbook(hit): + body = {k: v for k, v in hit["_source"].items()} + body.setdefault("tags", []) + + # convert any single tag to list of tags + if isinstance(body["tags"], six.string_types): + body["tags"] = [body["tags"]] + + if "_index" in hit: + body["region_id"] = hit["_index"][len("ms_runbooks_"):] + + latest_run = None + if 'inner_hits' in hit: + inner_hit = hit['inner_hits']['latest']['hits']['hits'] + if inner_hit: + latest_run = convert_run(inner_hit[0]) + + body["latest_run"] = latest_run + + body["id"] = hit["_id"] + return body diff --git a/runbook/api/v1/reader/__init__.py b/runbook/api/v1/reader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/runbook/api/v1/reader/run.py b/runbook/api/v1/reader/run.py new file mode 100644 index 0000000..a2e8f9e --- /dev/null +++ b/runbook/api/v1/reader/run.py @@ -0,0 +1,97 @@ +# Copyright 2016: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy + +import flask + +from runbook.api import utils +from runbook import storage + +API_TYPE = "reader" +bp = flask.Blueprint("runs", __name__) + + +def get_blueprints(): + return [["", bp]] + + +INNER_QUERY_WITH_PARENT = { + "has_parent": { + "inner_hits": { + "name": "parent", + "size": 1, + }, + "type": "runbook", + "query": { + "match_all": {} + } + } +} + +RUNS_QUERY = { + "query": { + "bool": { + "should": INNER_QUERY_WITH_PARENT, + } + } +} + + +@bp.route("/region//runbook_runs", methods=["GET"]) +@bp.route("/runbook_runs", methods=["GET"]) +@utils.check_regions(API_TYPE) +def handle_runs(region=None): + es = storage.get_elasticsearch(API_TYPE) + + if region is None: + region = "*" + index_name = "ms_runbooks_{}".format(region) + + query = copy.deepcopy(RUNS_QUERY) + + runbook_id = flask.request.args.get('runbook_id', '') + if runbook_id: + query["query"]["bool"]["should"]["has_parent"]["query"] = { + "ids": { + "values": runbook_id, + "type": "runbook", + } + } + + result = es.search(index=index_name, + doc_type="run", + body=query) + hit_list = [utils.convert_run(hit) for hit in result['hits']['hits']] + return flask.jsonify(hit_list) + + +@bp.route("/region//runbook_runs/", methods=["GET"]) +@utils.check_regions(API_TYPE) +def handle_single_run(region, run_id): + es = storage.get_elasticsearch(API_TYPE) + index_name = "ms_runbooks_{}".format(region) + + query = copy.deepcopy(RUNS_QUERY) + query["query"]["bool"]["must"] = {"ids": {"values": [run_id]}} + result = es.search(index=index_name, + doc_type="run", + body=query) + + hit_list = [utils.convert_run(hit) for hit in result['hits']['hits']] + if not hit_list: + flask.abort(404) + + return flask.jsonify(hit_list[0]) diff --git a/runbook/api/v1/reader/runbook_.py b/runbook/api/v1/reader/runbook_.py new file mode 100644 index 0000000..69a0c23 --- /dev/null +++ b/runbook/api/v1/reader/runbook_.py @@ -0,0 +1,121 @@ +# Copyright 2016: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy + +import flask + +from runbook.api import utils +from runbook import storage + +API_TYPE = "reader" +bp = flask.Blueprint("runbooks", __name__) + + +def get_blueprints(): + return [["", bp]] + + +INNER_QUERY_NO_RUNS = { + "bool": { + "must_not": [ + { + "has_child": { + "type": "run", + "query": { + "match_all": {} + } + } + } + ] + } +} + +INNER_QUERY_WITH_RUNS = { + "has_child": { + "inner_hits": { + "name": "latest", + "size": 1, + "sort": [{"created_at": {"order": "desc"}}] + }, + "type": "run", + "query": { + "match_all": {} + } + } +} + +RUNBOOK_QUERY = { + "query": { + "bool": { + "should": [ + INNER_QUERY_NO_RUNS, + INNER_QUERY_WITH_RUNS + ], + } + } +} + + +@bp.route("/runbooks", methods=["GET"]) +@bp.route("/region//runbooks", methods=["GET"]) +@utils.check_regions(API_TYPE) +def handle_runbooks(region=None): + es = storage.get_elasticsearch(API_TYPE) + + if region is None: + region = "*" + index_name = "ms_runbooks_{}".format(region) + + tags = flask.request.args.get('tags', '').split(',') + + query = copy.deepcopy(RUNBOOK_QUERY) + + # exclude deleted runbooks + query["query"]["bool"]["must_not"] = {"term": {"deleted": True}} + + # filter by tags, combining multiple 'term' queries effectively AND's + # supplied tags + tag_terms = [] + for tag in tags: + if tag: + tag_terms.append({"term": {"tags": tag}}) + if tag_terms: + query["query"]["bool"]["must"] = tag_terms + + result = es.search(index=index_name, + doc_type="runbook", + body=query) + hit_list = [utils.convert_runbook(hit) for hit in result['hits']['hits']] + return flask.jsonify(hit_list) + + +@bp.route("/region//runbooks/", methods=["GET"]) +@utils.check_regions(API_TYPE) +def handle_single_runbook(region, book_id): + es = storage.get_elasticsearch(API_TYPE) + index_name = "ms_runbooks_{}".format(region) + + query = copy.deepcopy(RUNBOOK_QUERY) + query["query"]["bool"]["must"] = {"ids": {"values": [book_id]}} + result = es.search(index=index_name, + doc_type="runbook", + body=query) + + hit_list = [utils.convert_runbook(hit) for hit in result['hits']['hits']] + if not hit_list: + flask.abort(404) + + return flask.jsonify(hit_list[0]) diff --git a/runbook/main_reader.py b/runbook/main_reader.py index 27084a4..cbbffc3 100644 --- a/runbook/main_reader.py +++ b/runbook/main_reader.py @@ -18,7 +18,8 @@ import flask from flask_helpers import routing -from runbook.api.v1 import runbook as runbook_api +from runbook.api.v1.reader import run as run_api +from runbook.api.v1.reader import runbook_ as runbook_api from runbook import config @@ -46,7 +47,7 @@ def handle_500(error): return flask.jsonify({"error": "Internal Server Error"}), 500 -for bp in [runbook_api]: +for bp in [runbook_api, run_api]: for url_prefix, blueprint in bp.get_blueprints(): app.register_blueprint(blueprint, url_prefix="/api/v1%s" % url_prefix) diff --git a/tests/unit/api/__init__.py b/tests/unit/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/api/v1/__init__.py b/tests/unit/api/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/api/v1/base.py b/tests/unit/api/v1/base.py new file mode 100644 index 0000000..89ed53c --- /dev/null +++ b/tests/unit/api/v1/base.py @@ -0,0 +1,63 @@ +# Copyright 2016: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import importlib +import json + +import mock +import testtools + + +TEST_CONFIG = { + "flask": { + "PORT": 5000, + "HOST": "0.0.0.0", + "DEBUG": True + }, + "backend": { + "type": "elastic", + "connection": [{"host": "127.0.0.1", "port": 9200}] + }, + "regions": [ + "region_one", + "region_two" + ] +} + + +class APITestCase(testtools.TestCase): + api_type = None + + def setUp(self): + super(APITestCase, self).setUp() + self.addCleanup(mock.patch.stopall) + + # NOTE(kzaitsev): mock all get_config for all the tests + self.patcher = mock.patch('runbook.config.get_config') + self.get_config = self.patcher.start() + self.get_config.return_value = TEST_CONFIG + + if self.api_type not in ["reader", "writer"]: + raise RuntimeError("Unknown api_type '{}'".format(self.api_type)) + + main = importlib.import_module("runbook.main_{}".format(self.api_type)) + self.client = main.app.test_client() + self.app = main.app + + def test_not_found(self): + resp = self.client.get('/404') + self.assertEqual({"error": "Not Found"}, + json.loads(resp.data.decode())) + self.assertEqual(404, resp.status_code) diff --git a/tests/unit/api/v1/reader/__init__.py b/tests/unit/api/v1/reader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/api/v1/reader/test_runbook_.py b/tests/unit/api/v1/reader/test_runbook_.py new file mode 100644 index 0000000..1658c42 --- /dev/null +++ b/tests/unit/api/v1/reader/test_runbook_.py @@ -0,0 +1,134 @@ +# Copyright 2016: Mirantis Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import json + +import elasticsearch +import mock + +from tests.unit.api.v1 import base + + +class RunbookTestCase(base.APITestCase): + api_type = "reader" + + correct_runbook = { + "name": "test", + "description": "test", + "type": "bash", + "runbook": "echo", + "tags": [], + "latest_run": None + } + + def test_get_no_region(self): + resp = self.client.get("/api/v1/region/region_zero/runbooks") + self.assertEqual(404, resp.status_code) + + resp = self.client.post("/api/v1/region/region_zero/runbooks") + self.assertEqual(405, resp.status_code) + + @mock.patch.object(elasticsearch.Elasticsearch, "search") + def test_get_runbooks(self, es_search): + es_search.return_value = { + "hits": {"hits": [ + { + "_id": "121", + "_source": self.correct_runbook, + }, + { + "_id": "122", + "_source": self.correct_runbook, + }, + { + "_id": "123", + "_source": self.correct_runbook, + }, + ]}, + } + resp = self.client.get("/api/v1/region/region_one/runbooks", + content_type="application/json") + self.assertEqual(200, resp.status_code) + resp_json = json.loads(resp.data.decode()) + expected = [] + for book_id in ["121", "122", "123"]: + data = copy.copy(self.correct_runbook) + data["id"] = book_id + expected.append(data) + self.assertEqual(expected, resp_json) + + es_search.assert_called_with(index="ms_runbooks_region_one", + doc_type="runbook", + body=mock.ANY) + + @mock.patch.object(elasticsearch.Elasticsearch, "search") + def test_get_single_runbook_bad_id(self, es_search): + es_search.return_value = { + "hits": {"hits": []} + } + resp = self.client.get("/api/v1/region/region_one/runbooks/123", + content_type="application/json") + self.assertEqual(404, resp.status_code) + + es_search.assert_called_with(index="ms_runbooks_region_one", + doc_type="runbook", + body=mock.ANY) + + @mock.patch.object(elasticsearch.Elasticsearch, "search") + def test_get_single_runbook(self, es_search): + es_search.return_value = { + "hits": {"hits": [ + { + "_id": "121", + "_source": self.correct_runbook, + }, + ]}, + } + resp = self.client.get("/api/v1/region/region_one/runbooks/123", + content_type="application/json") + self.assertEqual(200, resp.status_code) + + resp_json = json.loads(resp.data.decode()) + expected = copy.copy(self.correct_runbook) + expected["id"] = "121" + self.assertEqual(expected, resp_json) + + es_search.assert_called_with(index="ms_runbooks_region_one", + doc_type="runbook", + body=mock.ANY) + + @mock.patch.object(elasticsearch.Elasticsearch, "search") + def test_get_single_runbook_reg_two(self, es_search): + es_search.return_value = { + "hits": {"hits": [ + { + "_id": "123", + "_source": self.correct_runbook, + }, + ]}, + } + resp = self.client.get("/api/v1/region/region_two/runbooks/123", + content_type="application/json") + self.assertEqual(200, resp.status_code) + + resp_json = json.loads(resp.data.decode()) + expected = copy.copy(self.correct_runbook) + expected["id"] = "123" + self.assertEqual(expected, resp_json) + + es_search.assert_called_with(index="ms_runbooks_region_two", + doc_type="runbook", + body=mock.ANY)