Replace knit with generate_toc.py (#6279)

.github/workflows/quality.yml

@@ -287,12 +287,12 @@ jobs:
           path: |
             **/build/reports/**/*.*
 
-  knit:
+  docs:
-    name: Knit checks
+    name: Doc checks
     runs-on: ubuntu-latest
     # Allow all jobs on main and develop. Just one per PR.
     concurrency:
-      group: ${{ github.ref == 'refs/heads/main' && format('check-knit-main-{0}', github.sha) || github.ref == 'refs/heads/develop' && format('check-knit-develop-{0}', github.sha) || format('check-knit-{0}', github.ref) }}
+      group: ${{ github.ref == 'refs/heads/main' && format('check-docs-main-{0}', github.sha) || github.ref == 'refs/heads/develop' && format('check-docs-develop-{0}', github.sha) || format('check-docs-{0}', github.ref) }}
       cancel-in-progress: true
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -309,17 +309,9 @@ jobs:
       - name: Clone submodules
         if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'element-hq/element-x-android' }}
         run: git submodule update --init --recursive
-      - name: Use JDK 21
+      - name: Run docs check
-        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
+        # This is equivalent to `./gradlew checkDocs`, but we avoid having to install java and gradle
-        with:
+        run: python3 ./tools/docs/generate_toc.py --verify ./*.md docs/**/*.md
-          distribution: 'temurin' # See 'Supported distributions' for available options
-          java-version: '21'
-      - name: Configure gradle
-        uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2
-        with:
-          cache-read-only: ${{ github.ref != 'refs/heads/develop' }}
-      - name: Run Knit
-        run: ./gradlew knitCheck $CI_GRADLE_ARG_PROPERTIES
 
   # Note: to auto fix issues you can use the following command:
   # shellcheck -f diff <files> | git apply

CONTRIBUTING.md

@@ -16,7 +16,7 @@
   * [Code quality](#code-quality)
     * [detekt](#detekt)
     * [ktlint](#ktlint)
-    * [knit](#knit)
+    * [checkDocs](#checkdocs)
     * [lint](#lint)
   * [Unit tests](#unit-tests)
     * [konsist](#konsist)
@@ -123,13 +123,13 @@ Note that you can run
 
 For ktlint to fix some detected errors for you (you still have to check and commit the fix of course)
 
-#### knit
+#### checkDocs
 
-[knit](https://github.com/Kotlin/kotlinx-knit) is a tool which checks markdown files on the project. Also it generates/updates the table of content (toc) of the markdown files.
+`checkDocs` is a Gradle task which checks markdown files on the project to ensure their table of contents is up to date. It uses `tools/docs/generate_toc.py --verify` under the hood, and has a counterpart `generateDocsToc` task which runs `tools/docs/generate_toc.py` to update the table of contents of markdown files.
 
 So everytime the toc should be updated, just run
 <pre>
-./gradlew knit
+./gradlew generateDocsToc
 </pre>
 
 and commit the changes.
@@ -137,7 +137,7 @@ and commit the changes.
 The CI will check that markdown files are up to date by running
 
 <pre>
-./gradlew knitCheck
+./gradlew checkDocs
 </pre>
 
 #### lint

app/build.gradle.kts

@@ -33,7 +33,6 @@ plugins {
     alias(libs.plugins.kotlin.android)
     // When using precompiled plugins, we need to apply the firebase plugin like this
     id(libs.plugins.firebaseAppDistribution.get().pluginId)
-    alias(libs.plugins.knit)
     id("kotlin-parcelize")
     alias(libs.plugins.licensee)
     alias(libs.plugins.kotlin.serialization)
@@ -250,26 +249,6 @@ androidComponents {
     configureLicensesTasks(reportingExtension)
 }
 
-// Knit
-apply {
-    plugin("kotlinx-knit")
-}
-
-knit {
-    files = fileTree(project.rootDir) {
-        include(
-            "**/*.md",
-            "**/*.kt",
-            "*/*.kts",
-        )
-        exclude(
-            "**/build/**",
-            "*/.gradle/**",
-            "**/CHANGES.md",
-        )
-    }
-}
-
 setupDependencyInjection()
 
 dependencies {

build.gradle.kts

@@ -175,12 +175,23 @@ tasks.register("runQualityChecks") {
         tasks.findByName("ktlintCheck")?.let { dependsOn(it) }
         // tasks.findByName("buildHealth")?.let { dependsOn(it) }
     }
-    dependsOn(":app:knitCheck")
+    dependsOn("checkDocs")
-
     // Make sure all checks run even if some fail
     gradle.startParameter.isContinueOnFailure = true
 }
 
+// Register Markdown documentation check task.
+tasks.register("checkDocs", Exec::class.java) {
+    inputs.files("./*.md", "docs/**/*.md")
+    commandLine("python3", "tools/docs/generate_toc.py", "--verify", *inputs.files.map { it.path }.toTypedArray())
+}
+
+// Register Markdown documentation TOC generation task.
+tasks.register("generateDocsToc", Exec::class.java) {
+    inputs.files("./*.md", "docs/**/*.md")
+    commandLine("python3", "tools/docs/generate_toc.py", *inputs.files.map { it.path }.toTypedArray())
+}
+
 // Make sure to delete old screenshots before recording new ones
 subprojects {
     val snapshotsDir = File("${project.projectDir}/src/test/snapshots")

docs/install_from_github_release.md

@@ -10,8 +10,8 @@ This document explains how to install Element X Android from a Github Release.
   * [I already have the application on my phone](#i-already-have-the-application-on-my-phone)
 * [Installing from the App Bundle](#installing-from-the-app-bundle)
   * [Requirements](#requirements)
-  * [Steps](#steps)
+  * [Steps](#steps-1)
-  * [I already have the application on my phone](#i-already-have-the-application-on-my-phone)
+  * [I already have the application on my phone](#i-already-have-the-application-on-my-phone-1)
 
 <!--- END -->
 

docs/installing_from_ci.md

@@ -2,11 +2,11 @@
 
 <!--- TOC -->
 
-  * [Installing from GitHub](#installing-from-github)
+* [Installing from GitHub](#installing-from-github)
-    * [Create a GitHub token](#create-a-github-token)
+  * [Create a GitHub token](#create-a-github-token)
-  * [Provide artifact URL](#provide-artifact-url)
+* [Provide artifact URL](#provide-artifact-url)
-  * [Next steps](#next-steps)
+* [Next steps](#next-steps)
-  * [Future improvement](#future-improvement)
+* [Future improvement](#future-improvement)
 
 <!--- END -->
 

docs/integration_tests.md

@@ -8,7 +8,7 @@
 * [Stop Synapse](#stop-synapse)
 * [Troubleshoot](#troubleshoot)
   * [Android Emulator does cannot reach the homeserver](#android-emulator-does-cannot-reach-the-homeserver)
-  * [Tests partially run but some fail with "Unable to contact localhost:8080"](#tests-partially-run-but-some-fail-with-"unable-to-contact-localhost8080")
+  * [Tests partially run but some fail with "Unable to contact localhost:8080"](#tests-partially-run-but-some-fail-with-unable-to-contact-localhost8080)
   * [virtualenv command fails](#virtualenv-command-fails)
 
 <!--- END -->

docs/notifications.md

@@ -5,11 +5,11 @@ This document aims to describe how Element android displays notifications to the
 <!--- TOC -->
 
 * [Prerequisites Knowledge](#prerequisites-knowledge)
-  * [How does a matrix client get a message from a homeserver?](#how-does-a-matrix-client-get-a-message-from-a-homeserver?)
+  * [How does a matrix client get a message from a homeserver?](#how-does-a-matrix-client-get-a-message-from-a-homeserver)
   * [How does a mobile app receives push notification](#how-does-a-mobile-app-receives-push-notification)
   * [Push VS Notification](#push-vs-notification)
   * [Push in the matrix federated world](#push-in-the-matrix-federated-world)
-  * [How does the homeserver know when to notify a client?](#how-does-the-homeserver-know-when-to-notify-a-client?)
+  * [How does the homeserver know when to notify a client?](#how-does-the-homeserver-know-when-to-notify-a-client)
   * [Push vs privacy, and mitigation](#push-vs-privacy-and-mitigation)
   * [Background processing limitations](#background-processing-limitations)
 * [Element Notification implementations](#element-notification-implementations)

docs/pull_request.md

@@ -3,23 +3,23 @@
 <!--- TOC -->
 
 * [Introduction](#introduction)
-* [Who should read this document?](#who-should-read-this-document?)
+* [Who should read this document?](#who-should-read-this-document)
 * [Submitting PR](#submitting-pr)
-  * [Who can submit pull requests?](#who-can-submit-pull-requests?)
+  * [Who can submit pull requests?](#who-can-submit-pull-requests)
     * [Humans](#humans)
-      * [Draft PR?](#draft-pr?)
+      * [Draft PR?](#draft-pr)
       * [Base branch](#base-branch)
       * [PR Review Assignment](#pr-review-assignment)
       * [PR review time](#pr-review-time)
       * [Re-request PR review](#re-request-pr-review)
-      * [When create split PR?](#when-create-split-pr?)
+      * [When create split PR?](#when-create-split-pr)
       * [Avoid fixing other unrelated issue in a big PR](#avoid-fixing-other-unrelated-issue-in-a-big-pr)
     * [Bots](#bots)
       * [Renovate](#renovate)
       * [Gradle wrapper](#gradle-wrapper)
       * [Sync analytics plan](#sync-analytics-plan)
 * [Reviewing PR](#reviewing-pr)
-  * [Who can review pull requests?](#who-can-review-pull-requests?)
+  * [Who can review pull requests?](#who-can-review-pull-requests)
   * [What to have in mind when reviewing a PR](#what-to-have-in-mind-when-reviewing-a-pr)
   * [Rules](#rules)
     * [Check the form](#check-the-form)
@@ -29,7 +29,7 @@
       * [Check the commit](#check-the-commit)
       * [Check the substance](#check-the-substance)
       * [Make a dedicated meeting to review the PR](#make-a-dedicated-meeting-to-review-the-pr)
-  * [What happen to the issue(s)?](#what-happen-to-the-issues?)
+  * [What happen to the issue(s)?](#what-happen-to-the-issues)
   * [Merge conflict](#merge-conflict)
   * [When and who can merge PR](#when-and-who-can-merge-pr)
     * [Merge type](#merge-type)

gradle/libs.versions.toml

@@ -266,7 +266,6 @@ paparazzi = "app.cash.paparazzi:2.0.0-alpha04"
 roborazzi = { id = "io.github.takahirom.roborazzi", version.ref = "roborazzi" }
 sqldelight = { id = "app.cash.sqldelight", version.ref = "sqldelight" }
 firebaseAppDistribution = { id = "com.google.firebase.appdistribution", version.ref = "firebaseAppDistribution" }
-knit = { id = "org.jetbrains.kotlinx.knit", version = "0.5.1" }
 sonarqube = "org.sonarqube:7.2.3.7755"
 licensee = "app.cash.licensee:1.14.1"
 compose-compiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" }

tools/docs/generate_toc.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""Generate or verify markdown TOCs between HTML markers."""
+
+from __future__ import annotations
+
+import argparse
+import difflib
+import glob
+import re
+import sys
+from pathlib import Path
+
+TOC_MARKER = "<!--- TOC -->"
+END_MARKER = "<!--- END -->"
+HEADER_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
+FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})")
+
+
+def parse_headers(content: str) -> list[tuple[int, str]]:
+    """Extract markdown headers after the first END marker, excluding fenced code blocks."""
+    end_index = content.find(END_MARKER)
+    if end_index == -1:
+        return []
+
+    scan_region = content[end_index + len(END_MARKER) :]
+    headers: list[tuple[int, str]] = []
+    in_fence = False
+    fence_char = ""
+    fence_len = 0
+
+    for line in scan_region.splitlines():
+        fence_match = FENCE_RE.match(line)
+        if fence_match:
+            fence = fence_match.group(1)
+            if not in_fence:
+                in_fence = True
+                fence_char = fence[0]
+                fence_len = len(fence)
+            elif fence[0] == fence_char and len(fence) >= fence_len:
+                in_fence = False
+            continue
+
+        if in_fence:
+            continue
+
+        match = HEADER_RE.match(line)
+        if not match:
+            continue
+
+        level = len(match.group(1))
+        text = re.sub(r"\s+#+\s*$", "", match.group(2)).strip()
+        if text:
+            headers.append((level, text))
+
+    return headers
+
+
+def _slugify(text: str) -> str:
+    """Generate a markdown anchor similar to GitHub style."""
+    anchor = text.lower().strip()
+    anchor = re.sub(r"[^\w\s-]", "", anchor)
+    anchor = re.sub(r"\s+", "-", anchor)
+    anchor = re.sub(r"-+", "-", anchor).strip("-")
+    return anchor
+
+
+def generate_toc(headers: list[tuple[int, str]]) -> str:
+    """Generate markdown TOC content from parsed headers."""
+    if not headers:
+        return ""
+
+    min_level = min(level for level, _ in headers)
+    slug_counts: dict[str, int] = {}
+    toc_lines: list[str] = []
+
+    for level, text in headers:
+        base_slug = _slugify(text)
+        count = slug_counts.get(base_slug, 0)
+        slug_counts[base_slug] = count + 1
+        slug = base_slug if count == 0 else f"{base_slug}-{count}"
+
+        indent = "  " * (level - min_level)
+        toc_lines.append(f"{indent}* [{text}](#{slug})")
+
+    return "\n".join(toc_lines)
+
+
+def replace_toc_section(content: str, new_toc: str) -> str:
+    """Replace TOC block content between TOC and END markers."""
+    toc_start = content.find(TOC_MARKER)
+    if toc_start == -1:
+        raise ValueError("TOC marker not found")
+
+    toc_end = content.find(END_MARKER, toc_start + len(TOC_MARKER))
+    if toc_end == -1:
+        raise ValueError("END marker not found after TOC marker")
+
+    replacement = f"{TOC_MARKER}\n\n{new_toc}\n\n{END_MARKER}"
+    return content[:toc_start] + replacement + content[toc_end + len(END_MARKER) :]
+
+
+def build_expected_content(content: str) -> str:
+    """Build the expected markdown content after TOC regeneration."""
+    headers = parse_headers(content)
+    toc = generate_toc(headers)
+    return replace_toc_section(content, toc)
+
+
+def resolve_markdown_files(inputs: list[str]) -> list[Path]:
+    """Resolve CLI arguments to a de-duplicated ordered list of markdown files."""
+    files: list[Path] = []
+    seen: set[Path] = set()
+
+    def add_path(candidate: Path) -> None:
+        resolved = candidate.resolve()
+        if resolved.suffix.lower() != ".md" or not resolved.is_file() or resolved in seen:
+            return
+        seen.add(resolved)
+        files.append(resolved)
+
+    for item in inputs:
+        path = Path(item)
+        if path.exists():
+            if path.is_file():
+                add_path(path)
+            elif path.is_dir():
+                for md_file in sorted(path.rglob("*.md")):
+                    add_path(md_file)
+            continue
+
+        for matched in sorted(glob.glob(item, recursive=True)):
+            add_path(Path(matched))
+
+    return files
+
+
+def verify_file(path: Path) -> bool:
+    """Verify whether a file already contains the expected TOC."""
+    content = path.read_text(encoding="utf-8")
+
+    if TOC_MARKER not in content or END_MARKER not in content:
+        print(f"SKIP    | {path} (missing TOC markers)")
+        return True
+
+    expected = build_expected_content(content)
+    if expected == content:
+        print(f"OK      | {path}")
+        return True
+
+    print(f"OUTDATED| {path}", file=sys.stderr)
+    diff = difflib.unified_diff(
+        content.splitlines(),
+        expected.splitlines(),
+        fromfile=f"{path} (current)",
+        tofile=f"{path} (expected)",
+        lineterm="",
+    )
+    for line in diff:
+        print(line, file=sys.stderr)
+    return False
+
+
+def update_file(path: Path) -> bool:
+    """Regenerate and write TOC in a markdown file."""
+    content = path.read_text(encoding="utf-8")
+
+    if TOC_MARKER not in content or END_MARKER not in content:
+        print(f"SKIP    | {path} (missing TOC markers)")
+        return True
+
+    updated = build_expected_content(content)
+    if updated == content:
+        print(f"UNCHANGED| {path}")
+        return True
+
+    path.write_text(updated, encoding="utf-8")
+    print(f"UPDATED | {path}")
+    return True
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Generate markdown TOCs between <!--- TOC --> and <!--- END --> markers."
+    )
+    parser.add_argument("markdown_files", nargs="+", help="Markdown files, directories, or glob patterns")
+    parser.add_argument(
+        "--verify",
+        action="store_true",
+        help="Check files without modifying them; returns non-zero when TOC is outdated.",
+    )
+    args = parser.parse_args()
+
+    files = resolve_markdown_files(args.markdown_files)
+    if not files:
+        print("No markdown files were resolved from input arguments.", file=sys.stderr)
+        return 1
+
+    results = [verify_file(path) if args.verify else update_file(path) for path in files]
+    return 0 if all(results) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+