diff --git a/.circleci/template.yml b/.circleci/template.yml index 96c88efaa9e..d42dc533c3e 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -490,7 +490,7 @@ jobs: - add_ssh_keys - run: name: Install tools for building docs - command: tools/install-mkdocs.sh + command: tools/install-docs-tools.sh - run: name: Build the docs command: tools/build-docs.sh @@ -523,7 +523,6 @@ jobs: command: | ssh-keyscan github.com >> ~/.ssh/known_hosts git fetch git@github.com:esl/MongooseDocs.git gh-pages:gh-pages - pip3 install mike mike deploy $DOCS_TAG --remote git@github.com:esl/MongooseDocs.git --branch gh-pages --push build_in_docker: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4ffd0bf09b0..2b9fd85c10b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,6 +22,7 @@ Write type specifications and function signatures because they're remarkably hel Place technical documentaion in the directory `doc`. Find or create a suitable place for your material in terms of which page and section you write it in. Where documentation is placed is paramount, otherwise regardless of its quality, it will be harder to find it if at all. +To see your changes rendered, build the site locally as [described in the docs](https://esl.github.io/MongooseDocs/latest/developers-guide/Building-the-documentation/). What makes a good comment? Write about why something is done a certain way. diff --git a/Makefile b/Makefile index ad0f7918aa9..746ac7adf1e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: rel docs +.PHONY: rel docs docs-serve # GNU Make is required. Use gmake on BSD systems. @@ -85,6 +85,11 @@ docs: xeplist ## if build-docs.sh fails, mim_docs container remains running, so we can exec into it and debug ## ################################################################################################## docker run --rm -d -v "$(PWD):/mim" -w /mim --name mim_docs cimg/python:3.11.5-node sleep infinity - docker exec mim_docs tools/install-mkdocs.sh &>/dev/null + docker exec mim_docs tools/install-docs-tools.sh &>/dev/null docker exec mim_docs tools/build-docs.sh &>/dev/null docker stop mim_docs + +## Serve the docs locally with live reload, without going through Docker. +## Requires `pip3 install zensical`. +docs-serve: xeplist + zensical serve diff --git a/doc/css/custom.css b/doc/css/custom.css index c3fbe2024bf..ddb06e4e252 100644 --- a/doc/css/custom.css +++ b/doc/css/custom.css @@ -7,3 +7,8 @@ img#mim-readme-logo { --md-primary-fg-color: #459bb6; --md-accent-fg-color: #0554b8; } + +/* dark theme */ +[data-md-color-scheme="slate"] { + --md-accent-fg-color: #7ab5ff; +} diff --git a/doc/developers-guide/Building-the-documentation.md b/doc/developers-guide/Building-the-documentation.md new file mode 100644 index 00000000000..988bf22c068 --- /dev/null +++ b/doc/developers-guide/Building-the-documentation.md @@ -0,0 +1,34 @@ +# Building the documentation + +These docs live in `doc/` and are built with [Zensical](https://zensical.org/), configured by `mkdocs.yml` in the repository root - navigation, theme and Markdown extensions all come from that file. + +## Preview while you write + +```sh +pip3 install zensical +zensical serve +``` + +The site shows up at [http://localhost:8000](http://localhost:8000) and reloads whenever you save a file. +`make docs-serve` runs the same thing, but regenerates the supported XEP lists first with the [xep-tool](xep_tool.md), which reads compiled modules from `_build/prod/lib` - so use it only if you have built the release. + +The GraphQL API reference pages will look empty: they are generated by the two `npx spectaql` commands in `tools/build-docs.sh`. + +## Before you push + +```sh +zensical build --strict +``` + +Strict mode turns warnings into errors, most usefully links to pages and anchors that do not exist. +CI runs the same command and the build takes a few seconds, so there is no reason to find out on CI instead. +If the results look stale, add `-c` to drop the build cache. + +`make docs` is the whole CI build - XEP lists, GraphQL pages and site - inside a Docker container, so it needs no Python or Node on your machine. +When it fails, the `mim_docs` container is deliberately left running for you to `docker exec -it mim_docs bash` into. + +## Writing + +Include another file with `--8<-- "path/to/file.md"`, relative to `doc/`. +Diagrams go in a ` ```mermaid ` fence. +Content tabs, admonitions and highlighted code blocks with line numbers are available too. diff --git a/doc/listeners/listen-http.md b/doc/listeners/listen-http.md index 8124282c045..8910260eed2 100644 --- a/doc/listeners/listen-http.md +++ b/doc/listeners/listen-http.md @@ -137,7 +137,7 @@ The default value is 1 hour. Requires no additional options other than the [common handler options](#common-handler-options) in the listener section. In order to collect useful metrics, a `[prometheus]` section has to be added in [the instrumentation section](../configuration/instrumentation.md#). -The default configuration available with MongooseIM is shown in [Example 7](#example-7-prometheus) below. +The default configuration available with MongooseIM is shown in [Example 5](#example-5-prometheus) below. ## Transport options diff --git a/doc/migrations/5.0.0_5.1.0.md b/doc/migrations/5.0.0_5.1.0.md index 0ca24640b79..d6053c47b1b 100644 --- a/doc/migrations/5.0.0_5.1.0.md +++ b/doc/migrations/5.0.0_5.1.0.md @@ -9,7 +9,7 @@ The options `tls.verify_peer` and `tls.verify_mode` are replaced with just [`tls * `verify_peer = false` (the old default for C2S and HTTP listeners) should be replaced with `verify_mode = "none"`. * `verify_peer = true` should be replaced with `verify_mode = "peer"` or just removed, as this is the new default. -There is a new, simplified configuration format for `mongoose_client_api`. You need to change the `listen` section unless you have disabled the client API in your configuration file. Consult the [option description](../listeners/listen-http.md#handler-types-rest-api-client-mongoose_client_api) and the [example configuration](../listeners/listen-http.md#example-6-client-rest-api) for details. +There is a new, simplified configuration format for `mongoose_client_api`. You need to change the `listen` section unless you have disabled the client API in your configuration file. Consult the [option description](../listeners/listen-http.md) and the [example configuration](../listeners/listen-http.md) for details. ### Section `acl` diff --git a/doc/migrations/5.1.0_6.0.0.md b/doc/migrations/5.1.0_6.0.0.md index 93df2329d1b..32a711f9d82 100644 --- a/doc/migrations/5.1.0_6.0.0.md +++ b/doc/migrations/5.1.0_6.0.0.md @@ -1,7 +1,7 @@ ## Module configuration * The `mod_mam_meta` module is now named `mod_mam` for simplicity, so if you are using this module, you need to update the module name in `mongooseim.toml`. -* `mod_commands`, `mod_inbox_commands`, `mod_muc_commands` and `mod_muc_light_commands` are removed. Their functionality is now fully covered by [`mongoose_admin_api`](../listeners/listen-http.md/#handler-types-rest-api-admin-mongoose_admin_api). You need to delete these modules from `mongooseim.toml`. +* `mod_commands`, `mod_inbox_commands`, `mod_muc_commands` and `mod_muc_light_commands` are removed. Their functionality is now fully covered by [`mongoose_admin_api`](../listeners/listen-http.md). You need to delete these modules from `mongooseim.toml`. ## Metrics @@ -11,8 +11,8 @@ The `mod_mam` backend module is now named `mod_mam_pm` for consistency with `mod The whole REST API has been unified and simplified. There are now only two REST API handlers that you can configure in the `listen` section of `mongooseim.toml`: -- [`mongoose_admin_api`](../listeners/listen-http.md/#handler-types-rest-api-admin-mongoose_admin_api) handles the administrative API, -- [`mongoose_client_api`](../listeners/listen-http.md/#handler-types-rest-api-client-mongoose_client_api) handles the client-facing API. +- [`mongoose_admin_api`](../listeners/listen-http.md) handles the administrative API, +- [`mongoose_client_api`](../listeners/listen-http.md) handles the client-facing API. You need to remove the references to the obsolete handlers (`mongoose_api_client`, `mongoose_api_admin`, `mongoose_api`, `mongoose_domain_handler`) from your configuration file. diff --git a/doc/migrations/6.5.0_6.6.0.md b/doc/migrations/6.5.0_6.6.0.md index d66686b241f..9c1a70994da 100644 --- a/doc/migrations/6.5.0_6.6.0.md +++ b/doc/migrations/6.5.0_6.6.0.md @@ -4,7 +4,7 @@ The following features are removed or deprecated. ### MS SQL and ODBC are no longer supported -The Microsoft SQL Server database is no longer supported, and you cannot set the [RDBMS driver](../configuration/outgoing-connections.md/#outgoing_poolsrdbmsconnectiondriver) to `odbc` anymore. +The Microsoft SQL Server database is no longer supported, and you cannot set the [RDBMS driver](../configuration/outgoing-connections.md#outgoing_poolsrdbmsconnectiondriver) to `odbc` anymore. In case you used this database, you need to migrate to another supported RDBMS, e.g. MySQL, MariaDB, PostgreSQL or CockroachDB. ### Removal of admin REST API diff --git a/doc/tutorials/jingle.md b/doc/tutorials/jingle.md index 13266435fbe..14a8f21bf25 100644 --- a/doc/tutorials/jingle.md +++ b/doc/tutorials/jingle.md @@ -59,114 +59,109 @@ For small groups (typically up to four participants for video), each participant * Strict codec agreement: all participants must use compatible audio/video formats. * Client-side mixing: each participant must mix incoming audio and video streams locally. -```puml -@startuml -rectangle "full mesh" { - rectangle " <&phone*5>\n Peer 1" as p1 - rectangle " <&phone*5>\n Peer 2" as p2 - rectangle " <&phone*5>\n Peer 3" as p3 - rectangle " <&phone*5>\n Peer 4" as p4 - - p1 <-down-> p2 - p1 <-down-> p3 - p1 <-down-> p4 - p2 <-right-> p3 - p2 <-down-> p4 - p3 <-down-> p4 -} -@enduml +```mermaid +flowchart TB + subgraph mesh["full mesh"] + p1["📞 Peer 1"] + p2["📞 Peer 2"] + p3["📞 Peer 3"] + p4["📞 Peer 4"] + p1 <--> p2 + p1 <--> p3 + p1 <--> p4 + p2 <--> p3 + p2 <--> p4 + p3 <--> p4 + end ``` [STUN]/[TURN] servers can still be used to establish peer-to-peer connections, similar to one-to-one calls. [XEP-0272] also suggests using `RTP relays` to reduce upstream bandwidth usage. -```puml -@startuml -rectangle "relay example" { - rectangle " <&phone*5>\n Peer 1" as p1 - rectangle " " as p2i { - rectangle " <&phone*5>\n Peer 2" as p2 - rectangle "RTP relay" as r1 #lightblue - } - rectangle " <&phone*5>\n Peer 3" as p3 - rectangle " <&phone*5>\n Peer 4" as p4 - - note bottom of p2i: Peer 2 application infrastructure - - p1 -down-> p2 #violet - p1 <-down-> p3 - p1 <-down-> p4 - p2 -right-> r1 #lightblue - r1 -up-> p1 #lightblue - r1 -right-> p3 #lightblue - r1 -down-> p4 #lightblue - p3 -right-> p2 #violet - p3 <-down-> p4 - p4 -up-> p2 #violet -} -@enduml +```mermaid +flowchart TB + subgraph relay["relay example"] + p1["📞 Peer 1"] + subgraph p2i["Peer 2 application infrastructure"] + p2["📞 Peer 2"] + r1["RTP relay"] + end + p3["📞 Peer 3"] + p4["📞 Peer 4"] + p1 --> p2 + p1 <--> p3 + p1 <--> p4 + p2 --> r1 + r1 --> p1 + r1 --> p3 + r1 --> p4 + p3 --> p2 + p3 <--> p4 + p4 --> p2 + end + style r1 fill:#0288d133,stroke:#0288d1,stroke-width:2px + linkStyle 3,4,5,6 stroke:#0288d1,stroke-width:2px + linkStyle 0,7,9 stroke:#ba68c8,stroke-width:2px ``` To further reduce downstream bandwidth and CPU usage, `content mixers` can be introduced. -```puml -@startuml -rectangle "mixer example" { - rectangle " <&phone*5>\n Peer 1" as p1 - rectangle " " as p2i { - rectangle " <&phone*5>\n Peer 2" as p2 - rectangle "Content mixer" as m1 #lightgreen - } - rectangle " <&phone*5>\n Peer 3" as p3 - rectangle " <&phone*5>\n Peer 4" as p4 - - note bottom of p2i: Peer 2 application infrastructure - - p1 -down-> m1 #lightgreen - p1 <-down-> p3 - p1 <-down-> p4 - m1 -right-> p2 #lightgreen - p2 -up-> p1 #violet - p2 -right-> p3 #violet - p2 -down-> p4 #violet - p3 -left-> m1 #lightgreen - p3 <-down-> p4 - p4 -up-> m1 #lightgreen -} -@enduml +```mermaid +flowchart TB + subgraph mixer["mixer example"] + p1["📞 Peer 1"] + subgraph p2i["Peer 2 application infrastructure"] + p2["📞 Peer 2"] + m1["Content mixer"] + end + p3["📞 Peer 3"] + p4["📞 Peer 4"] + p1 --> m1 + p1 <--> p3 + p1 <--> p4 + m1 --> p2 + p2 --> p1 + p2 --> p3 + p2 --> p4 + p3 --> m1 + p3 <--> p4 + p4 --> m1 + end + style m1 fill:#43a04733,stroke:#43a047,stroke-width:2px + linkStyle 0,3,7,9 stroke:#43a047,stroke-width:2px + linkStyle 4,5,6 stroke:#ba68c8,stroke-width:2px ``` However, to ensure interoperability, each client must independently allocate its own `RTP relay` and `content mixer`. -```puml -@startuml -rectangle "complex mesh example" { - - rectangle " <&phone*5>\n Peer 1" as p1 - rectangle " " as p2i { - rectangle " <&phone*5>\n Peer 2" as p2 - rectangle "Content mixer" as m1 #lightgreen - rectangle "RTP relay" as r1 #lightblue - } - rectangle " <&phone*5>\n Peer 3" as p3 - rectangle " <&phone*5>\n Peer 4" as p4 - - note bottom of p2i: Peer 2 application infrastructure - - p1 -down-> m1 #lightgreen - p1 <-down-> p3 - p1 <-down-> p4 - m1 -right-> p2 #lightgreen - p2 -right-> r1 #lightblue - r1 -up-> p1 #lightblue - r1 -right-> p3 #lightblue - r1 -down-> p4 #lightblue - p3 -left-> m1 #lightgreen - p3 <-down-> p4 - p4 -up-> m1 #lightgreen -} -@enduml +```mermaid +flowchart TB + subgraph complex["complex mesh example"] + p1["📞 Peer 1"] + subgraph p2i["Peer 2 application infrastructure"] + p2["📞 Peer 2"] + m1["Content mixer"] + r1["RTP relay"] + end + p3["📞 Peer 3"] + p4["📞 Peer 4"] + p1 --> m1 + p1 <--> p3 + p1 <--> p4 + m1 --> p2 + p2 --> r1 + r1 --> p1 + r1 --> p3 + r1 --> p4 + p3 --> m1 + p3 <--> p4 + p4 --> m1 + end + style m1 fill:#43a04733,stroke:#43a047,stroke-width:2px + style r1 fill:#0288d133,stroke:#0288d1,stroke-width:2px + linkStyle 0,3,8,10 stroke:#43a047,stroke-width:2px + linkStyle 4,5,6,7 stroke:#0288d1,stroke-width:2px ``` This results in inefficiencies, as the same optimization components are duplicated for each participant. In practice, this approach often evolves from a mesh topology into a star topology. @@ -178,31 +173,31 @@ Large conferences require a centralized architecture. In this model: * A `focus agent` is responsible for allocating and managing these resources. * Each participant establishes an individual Jingle session with the `focus agent`, similar to a one-to-one call. -```puml -@startuml -rectangle "conference example" { - rectangle " <&phone*5>\n Peer 1" as p1 - rectangle " <&phone*5>\n Peer 2" as p2 - rectangle " <&phone*5>\n Peer 3" as p3 - rectangle " <&phone*5>\n Peer 4" as p4 - rectangle " " as fai { - rectangle "Content mixer" as m1 #lightgreen - rectangle "RTP relay" as r1 #lightblue - } - - note bottom of fai: Focus agent application infrastructure - - p1 -down-> m1 #lightgreen - p2 -down-> m1 #lightgreen - p3 -up-> m1 #lightgreen - p4 -up-> m1 #lightgreen - m1 -right-> r1 #lightblue - r1 -up-> p1 #lightblue - r1 -up-> p2 #lightblue - r1 -down-> p3 #lightblue - r1 -down-> p4 #lightblue -} -@enduml +```mermaid +flowchart TB + subgraph conference["conference example"] + p1["📞 Peer 1"] + p2["📞 Peer 2"] + p3["📞 Peer 3"] + p4["📞 Peer 4"] + subgraph fai["Focus agent application infrastructure"] + m1["Content mixer"] + r1["RTP relay"] + end + p1 --> m1 + p2 --> m1 + p3 --> m1 + p4 --> m1 + m1 --> r1 + r1 --> p1 + r1 --> p2 + r1 --> p3 + r1 --> p4 + end + style m1 fill:#43a04733,stroke:#43a047,stroke-width:2px + style r1 fill:#0288d133,stroke:#0288d1,stroke-width:2px + linkStyle 0,1,2,3 stroke:#43a047,stroke-width:2px + linkStyle 4,5,6,7,8 stroke:#0288d1,stroke-width:2px ``` Conference management is partially defined by: @@ -266,7 +261,7 @@ sequenceDiagram participant Bob participant TURN as STUN/TURN Server - rect rgba(0, 255, 255, 0.5) + rect rgba(2, 136, 209, 0.15) note right of Alice: XEP-0215: External Service Discovery (optional) note right of Alice: Usually performed right after connection Alice->>XMPP: External Service Discovery IQ @@ -275,14 +270,14 @@ sequenceDiagram XMPP-->>Bob: STUN/TURN server address and credentials end - rect rgba(0, 255, 255, 0.5) + rect rgba(2, 136, 209, 0.15) note right of Alice: XEP-0353: Jingle Message Initiation (optional) Alice->>Bob: propose jingle-message Bob-->>Alice: ringing jingle-message Bob-->>Alice: proceed jingle-message end - rect rgba(255, 255, 0, 0.5) + rect rgba(255, 193, 7, 0.18) note right of Alice: ICE candidate gathering & Jingle session setup note right of Alice: XEP-0166: Jingle
XEP-0167: Jingle RTP Sessions
XEP-0176: Jingle ICE-UDP Transport Method Alice->>TURN: STUN binding request @@ -299,20 +294,20 @@ sequenceDiagram Alice-->>Bob: result IQ end - rect rgba(255, 255, 0, 0.5) + rect rgba(255, 193, 7, 0.18) note right of Alice: non-XMPP traffic Alice<<->>Bob: ICE connectivity checks Alice<<->>Bob: RTP/SRTP Media Session end - rect rgba(255, 255, 0, 0.5) + rect rgba(255, 193, 7, 0.18) note right of Alice: Session termination note right of Alice: XEP-0166: Jingle Alice->>Bob: jingle session-terminate IQ Bob-->>Alice: result IQ end - rect rgba(0, 255, 255, 0.5) + rect rgba(2, 136, 209, 0.15) note right of Alice: XEP-0353: Jingle Message Initiation (optional) Alice->>Bob: finish jingle-message Bob->>Alice: finish jingle-message diff --git a/doc/user-guide/Supported-standards.md b/doc/user-guide/Supported-standards.md index 0a500330b24..43de409e3d6 100644 --- a/doc/user-guide/Supported-standards.md +++ b/doc/user-guide/Supported-standards.md @@ -33,9 +33,7 @@ ## Supported XEPs -{% - include-markdown "./Supported-XEPs.md" -%} +--8<-- "user-guide/Supported-XEPs.md" ## Supported Open Extensions diff --git a/mkdocs.yml b/mkdocs.yml index 72e129be336..77f46154b90 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -36,23 +36,21 @@ extra: - icon: fontawesome/brands/github link: https://github.com/esl/MongooseIM plugins: - - include-markdown - - mermaid2: - version: 11.1.0 - - plantuml: - puml_url: https://www.plantuml.com/plantuml/ - search: separator: '[\s\-_,:!=\[\]()"/<>]+|\.(?!\d)|&[lg]t;' lang: en extra_css: [css/custom.css] markdown_extensions: + - pymdownx.snippets: + base_path: [doc] + check_paths: true - pymdownx.highlight: linenums: true - pymdownx.superfences: custom_fences: - name: mermaid class: mermaid - format: !!python/name:mermaid2.fence_mermaid + format: !!python/name:pymdownx.superfences.fence_code_format - admonition - pymdownx.tabbed: alternate_style: true @@ -193,6 +191,7 @@ nav: - 'Release/Installation configuration': 'developers-guide/release_config.md' - 'Bootstrap scripts': 'developers-guide/Bootstrap-Scripts.md' - 'Testing MongooseIM': 'developers-guide/Testing-MongooseIM.md' + - 'Building the documentation': 'developers-guide/Building-the-documentation.md' - 'Logging': 'developers-guide/logging.md' - 'Hooks and Handlers': 'developers-guide/Hooks-and-handlers.md' - 'Hooks description': 'developers-guide/hooks_description.md' diff --git a/tools/build-docs.sh b/tools/build-docs.sh index c3b25912d2f..7c6aff55938 100755 --- a/tools/build-docs.sh +++ b/tools/build-docs.sh @@ -2,29 +2,9 @@ set -e -retry() { - local max_attempts="$1" - shift - local attempt=1 - - while [ "$attempt" -le "$max_attempts" ]; do - if "$@"; then - return 0 - fi - echo "Attempt $attempt failed. Retrying..." >&2 - ((attempt++)) - done - - return 1 -} - ## Build GraphQL static docs npx spectaql -t doc/graphql-api -f admin-graphql-doc.html doc/graphql-api/Admin-GraphQL_spectaql.yml npx spectaql -C -J -t doc/graphql-api -f user-graphql-doc.html doc/graphql-api/User-GraphQL_spectaql.yml ## Build docs - -## Sometimes public plantuml server is too busy and mkdocs build fails, -## so we need to retry it a few times. Successfully built diagrams a cached locally, -## so next attempts will be faster and more likely to succeed. -retry 10 mkdocs build --strict +zensical build --strict diff --git a/tools/install-docs-tools.sh b/tools/install-docs-tools.sh new file mode 100755 index 00000000000..2ee83a0714a --- /dev/null +++ b/tools/install-docs-tools.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env sh + +set -e + +## spectaql (for building GraphQL static docs) +npm install --global spectaql + +## Zensical itself +pip3 install zensical + +## mike for Zensical +pip3 install git+https://github.com/squidfunk/mike.git diff --git a/tools/install-mkdocs.sh b/tools/install-mkdocs.sh deleted file mode 100755 index 0e7c3eadc35..00000000000 --- a/tools/install-mkdocs.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env sh - -set -e - -## Install spectaql for building GraphQL static docs -npm install --global cheerio@1.0.0-rc.12 spectaql - -## Install mkdocs and plugins -pip3 install mkdocs -pip3 install mkdocs-material -pip3 install mkdocs-include-markdown-plugin -pip3 install mkdocs_puml -pip3 install mkdocs-mermaid2-plugin