chore: rename docs directory and other adjustments (#10157)

* chore: rename docs directory and other adjustments

* fixes
This commit is contained in:
Stefano Fiorucci
2025-11-28 08:41:35 +01:00
committed by GitHub
parent ea515f53d1
commit af75774c3f
46 changed files with 104 additions and 141 deletions

View File

@@ -1,24 +1,15 @@
#!/bin/bash
# Usage: ./pydoc-markdown.sh [CONFIG_PATH]
#
# Generate documentation from pydoc-markdown config files.
#
# Examples:
# ./pydoc-markdown.sh # Uses default path: ../config/*
# ./pydoc-markdown.sh ../config/api/* # Uses custom path
# ./pydoc-markdown.sh /path/to/configs/* # Uses absolute path
# Generate API documentation from pydoc-markdown config files.
# Run from repo root using hatch run docs
set -e # Fails on any error in the following loop
set -e # Fails on any error in the following loop
# Set default config path or use provided parameter
CONFIG_PATH="${1:-../config/*}"
cd docs/pydoc
cd pydoc
rm -rf temp && mkdir temp
cd temp
echo "Processing config files in $CONFIG_PATH"
for file in $CONFIG_PATH ; do
for file in ../*.yml ; do
echo "Converting $file..."
pydoc-markdown "$file"
done

View File

@@ -6,7 +6,7 @@ on:
branches:
- main
paths:
- "docs/pydoc/config_docusaurus/**"
- "pydoc/**"
- "haystack/**"
- ".github/workflows/docusaurus_sync.yml"
@@ -33,12 +33,11 @@ jobs:
run: pip install hatch==${{ env.HATCH_VERSION }}
- name: Generate API reference for Docusaurus
# This command simply runs ./.github/utils/pydoc-markdown.sh with a specific config path
run: hatch run readme:sync "../config_docusaurus/*"
run: hatch run docs
- name: Sync generated API reference to docs folder
run: |
SOURCE_PATH="docs/pydoc/temp"
SOURCE_PATH="pydoc/temp"
DEST_PATH="docs-website/reference/haystack-api"
echo "Syncing from $SOURCE_PATH to $DEST_PATH"

2
.gitignore vendored
View File

@@ -84,7 +84,7 @@ instance/
.scrapy
# documentation
docs/pydoc/temp/
pydoc/temp/
# PyBuilder
target/

View File

@@ -323,16 +323,16 @@ executing some workflows on your changes, like automated tests, linting, formatt
If all goes well, at the bottom of your PR page you should see something like this, where all checks are green.
![Successful CI](docs/img/ci-success.png)
![Successful CI](images/ci-success.png)
If you see some red checks (like the following), then something didn't work, and action is needed from your side.
![Failed CI](docs/img/ci-failure-example.png)
![Failed CI](images/ci-failure-example.png)
Click on the failing test and see if there are instructions at the end of the logs of the failed test.
For example, in the case above, the CI will give you instructions on how to fix the issue.
![Logs of failed CI, with instructions for fixing the failure](docs/img/ci-failure-example-instructions.png)
![Logs of failed CI, with instructions for fixing the failure](images/ci-failure-example-instructions.png)
## Working from GitHub forks
@@ -341,7 +341,7 @@ In order for maintainers to be able to help you, we usually ask contributors to
To do so, please verify that "Allow edits and access to secrets by maintainers" on the PR preview page is checked
(you can check it later on the PR's sidebar once it's created).
![Allow access to your branch to maintainers](docs/img/first_time_contributor_enable_access.png)
![Allow access to your branch to maintainers](images/first_time_contributor_enable_access.png)
## Writing tests

View File

@@ -1,5 +1,5 @@
<div align="center">
<a href="https://haystack.deepset.ai/"><img src="https://raw.githubusercontent.com/deepset-ai/haystack/main/docs/img/banner.png" alt="Green logo of a stylized white 'H' with the text 'Haystack, by deepset.' Abstract green and yellow diagrams in the background."></a>
<a href="https://haystack.deepset.ai/"><img src="https://raw.githubusercontent.com/deepset-ai/haystack/main/images/banner.png" alt="Green logo of a stylized white 'H' with the text 'Haystack, by deepset.' Abstract green and yellow diagrams in the background."></a>
| | |
| ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
@@ -91,7 +91,7 @@ Use **deepset Studio** to visually create, deploy, and test your Haystack pipeli
👉 [Sign up](https://landing.deepset.ai/deepset-studio-signup)!
> [!TIP]
><img src="https://github.com/deepset-ai/haystack/raw/main/docs/img/deepset-platform-logo-alternative.jpeg" width=20%>
><img src="https://github.com/deepset-ai/haystack/raw/main/images/deepset-platform-logo-alternative.jpeg" width=20%>
>
> Are you looking for a managed solution that benefits from Haystack? [deepset AI Platform](https://www.deepset.ai/products-and-services/deepset-ai-platform?utm_campaign=developer-relations&utm_source=haystack&utm_medium=readme) is our fully managed, end-to-end platform to integrate LLMs with your data, which uses Haystack for the LLM pipelines architecture.

View File

@@ -1,57 +0,0 @@
# :ledger: Looking for the docs?
You can find Haystack's documentation at https://docs.haystack.deepset.ai/.
# :computer: How to update docs?
## Overview, Components, Pipeline Nodes, and Guides
You can find these docs on the Haystack Docs page: https://docs.haystack.deepset.ai/docs/get_started. If you want to contribute, and we welcome every contribution, do the following:
1. Make sure you're on the right version (check the version expanding list in the top left corner).
2. Use the "Suggest Edits" link you can find in the top right corner of every page.
3. Suggest a change right in the docs and click **Submit Suggested Edits**.
4. Optionally, leave us a comment and submit your change.
Once we take care of it, you'll get an email telling you the change's been merged, or not. If not, we'll give you the reason why.
Make sure to check our [Contribution Guidelines](https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md).
## Tutorials
The Tutorials live in a separate repo: https://github.com/deepset-ai/haystack-tutorials. For instructions on how to contribute to tutorials, see [Contributing to Tutorials](https://github.com/deepset-ai/haystack-tutorials/blob/main/Contributing.md#contributing-to-haystack-tutorials).
## API Reference
We use Pydoc-Markdown to create Markdown files from the docstrings in our code. There is a Github Action that regenerates the API pages with each commit.
If you want to generate a new Markdown file for a new Haystack module, create a `.yml` file in `docs/src/api/api` which configures how Pydoc-Markdown will generate the page and commit it to main.
All the updates to doctrings get pushed to documentation when you commit to the main branch.
### Configuration
Pydoc will read the configuration from a `.yml` file which is located under `/haystack/docs/_src/api/pydoc`. Our files contain three main sections:
- **loader**: A list of plugins that load API objects from python source files.
- **type**: Loader for python source files
- **search_path**: Location of source files
- **modules**: Module which are used for generating the markdown file
- **ignore_when_discovered**: Define which files should be ignored
- **processor**: A list of plugins that process API objects to modify their docstrings (e.g. to adapt them from a documentation format to Markdown or to remove items that should not be rendered into the documentation).
- **type: filter**: Filter for specific modules
- **documented_only**: Only documented API objects
- **do_not_filter_modules**: Do not filter module objects
- **skip_empty_modules**: Skip modules without content
- **renderer**: A plugin that produces the output files. We use a custom ReadmeRenderer based on the Markdown renderer. It makes sure the Markdown files comply with ReadMe requirements.
- **type**: Define the renderer which you want to use. We are using the ReadmeRenderer to make sure the files display properly in ReadMe.
- **excerpt**: Add a short description of the page. It shows up right below the page title.
- **category**: This is the ReadMe category ID to make sure the doc lands in the right section of Haystack docs.
- **title**: The title of the doc as it will appear on the website. Make sure you always add "API" at the end.
- **slug**: The page slug, each word should be separated with a dash.
- **order**: Pages are ordered alphabetically. This defines where in the TOC the page lands.
- markdown:
- **descriptive_class_title**: Remove the word "Object" from class titles.
- **descriptive_module_title**: Adding the word “Module” before the module name.
- **add_method_class_prefix**: Add the class name as a prefix to method names.
- **add_member_class_prefix**: Add the class name as a prefix to member names.
- **filename**: File name of the generated file, use underscores to separate each word.

View File

Before

Width:  |  Height:  |  Size: 175 KiB

After

Width:  |  Height:  |  Size: 175 KiB

View File

Before

Width:  |  Height:  |  Size: 131 KiB

After

Width:  |  Height:  |  Size: 131 KiB

View File

Before

Width:  |  Height:  |  Size: 102 KiB

After

Width:  |  Height:  |  Size: 102 KiB

View File

Before

Width:  |  Height:  |  Size: 50 KiB

After

Width:  |  Height:  |  Size: 50 KiB

View File

Before

Width:  |  Height:  |  Size: 101 KiB

After

Width:  |  Height:  |  Size: 101 KiB

View File

Before

Width:  |  Height:  |  Size: 154 KiB

After

Width:  |  Height:  |  Size: 154 KiB

39
pydoc/README.md Normal file
View File

@@ -0,0 +1,39 @@
# :ledger: Looking for the docs?
You can find Haystack's documentation at https://docs.haystack.deepset.ai/.
# API Reference
We use Pydoc-Markdown to create Markdown files from the docstrings in our code. There is a Github Action that regenerates the API pages when tags are pushed or when manually triggered.
If you want to generate a new Markdown file for a new Haystack module, create a `.yml` file in `pydoc` which configures how Pydoc-Markdown will generate the page and commit it to main.
All the updates to docstrings get pushed to documentation when a new version is released.
### Configuration
Pydoc will read the configuration from a `.yml` file which is located under `pydoc`. Our files contain three main sections:
- **loaders**: A list of plugins that load API objects from python source files.
- **type**: Loader for python source files (we use `haystack_pydoc_tools.loaders.CustomPythonLoader`)
- **search_path**: Location of source files (relative to the config file)
- **modules**: Modules which are used for generating the markdown file
- **ignore_when_discovered**: Define which files should be ignored
- **processors**: A list of plugins that process API objects to modify their docstrings (e.g. to adapt them from a documentation format to Markdown or to remove items that should not be rendered into the documentation).
- **type: filter**: Filter for specific modules
- **documented_only**: Only documented API objects
- **do_not_filter_modules**: Do not filter module objects
- **skip_empty_modules**: Skip modules without content
- **type: smart**: Smart processor for docstring processing
- **type: crossref**: Cross-reference processor
- **renderer**: A plugin that produces the output files. We use a custom DocusaurusRenderer based on the Markdown renderer. It makes sure the Markdown files comply with Docusaurus requirements.
- **type**: Define the renderer which you want to use. We are using the DocusaurusRenderer to make sure the files display properly in Docusaurus.
- **description**: Add a short description of the page. It shows up right below the page title.
- **id**: This is the Docusaurus page ID to make sure the doc lands in the right section of Haystack docs.
- **title**: The title of the doc as it will appear on the website. Make sure you always add "API" at the end.
- **markdown**:
- **descriptive_class_title**: Remove the word "Object" from class titles.
- **descriptive_module_title**: Adding the word "Module" before the module name.
- **add_method_class_prefix**: Add the class name as a prefix to method names.
- **add_member_class_prefix**: Add the class name as a prefix to member names.
- **filename**: File name of the generated file, use underscores to separate each word.

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/agents]
search_path: [../haystack/components/agents]
modules: ["agent", "state/state"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/audio]
search_path: [../haystack/components/audio]
modules: ["whisper_local", "whisper_remote"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/builders]
search_path: [../haystack/components/builders]
modules: ["answer_builder", "prompt_builder", "chat_prompt_builder"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/caching]
search_path: [../haystack/components/caching]
modules: ["cache_checker"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/classifiers]
search_path: [../haystack/components/classifiers]
modules: ["document_language_classifier", "zero_shot_document_classifier"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/connectors]
search_path: [../haystack/components/connectors]
modules: ["openapi_service", "openapi"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/converters]
search_path: [../haystack/components/converters]
modules:
[
"azure",

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/dataclasses]
search_path: [../haystack/dataclasses]
modules:
["answer", "byte_stream", "chat_message", "document", "image_content", "sparse_embedding", "streaming_chunk"]
ignore_when_discovered: ["__init__"]

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/document_stores/in_memory]
search_path: [../haystack/document_stores/in_memory]
modules: ["document_store"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/writers]
search_path: [../haystack/components/writers]
modules: ["document_writer"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/embedders]
search_path: [../haystack/components/embedders]
modules:
[
"azure_document_embedder",

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/evaluation]
search_path: [../haystack/evaluation]
modules:
[
"eval_run_result",

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/evaluators]
search_path: [../haystack/components/evaluators]
modules:
[
"answer_exact_match",

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/extractors]
search_path: [../haystack/components/extractors]
modules: [
"named_entity_extractor",
"llm_metadata_extractor",

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/fetchers]
search_path: [../haystack/components/fetchers]
modules: ["link_content"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/generators]
search_path: [../haystack/components/generators]
modules:
[
"azure",

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/converters/image]
search_path: [../haystack/components/converters/image]
modules:
[
"document_to_image",

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/joiners]
search_path: [../haystack/components/joiners]
modules: ["answer_joiner", "branch", "document_joiner", "list_joiner", "string_joiner"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/core/pipeline]
search_path: [../haystack/core/pipeline]
modules: ["async_pipeline","pipeline"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/preprocessors]
search_path: [../haystack/components/preprocessors]
modules: [
"csv_document_cleaner",
"csv_document_splitter",

View File

@@ -1,28 +1,25 @@
loaders:
- ignore_when_discovered:
- __init__
modules:
- haystack.components.query.query_expander
search_path:
- ../../../
type: haystack_pydoc_tools.loaders.CustomPythonLoader
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../haystack/components/query]
modules: ["query_expander"]
ignore_when_discovered: ["__init__"]
processors:
- do_not_filter_modules: false
documented_only: true
expression: null
skip_empty_modules: true
type: filter
- type: smart
- type: crossref
- type: filter
expression:
documented_only: true
do_not_filter_modules: false
skip_empty_modules: true
- type: smart
- type: crossref
renderer:
type: haystack_pydoc_tools.renderers.DocusaurusRenderer
description: Components for query processing and expansion.
title: Query
id: query-api
markdown:
add_member_class_prefix: false
add_method_class_prefix: true
classdef_code_block: false
descriptive_class_title: false
classdef_code_block: false
descriptive_module_title: true
add_method_class_prefix: true
add_member_class_prefix: false
filename: query_api.md
title: Query
type: haystack_pydoc_tools.renderers.DocusaurusRenderer

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/rankers]
search_path: [../haystack/components/rankers]
modules: [
"hugging_face_tei",
"lost_in_the_middle",

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/readers]
search_path: [../haystack/components/readers]
modules: ["extractive"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/retrievers]
search_path: [../haystack/components/retrievers]
modules:
[
"auto_merging_retriever",

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/routers]
search_path: [../haystack/components/routers]
modules:
[
"conditional_router",

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/samplers]
search_path: [../haystack/components/samplers]
modules: ["top_p"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/tools]
search_path: [../haystack/components/tools]
modules: ["tool_invoker"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/tools]
search_path: [../haystack/tools]
modules:
["tool", "from_function", "component_tool", "toolset"]
ignore_when_discovered: ["__init__"]

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/utils]
search_path: [../haystack/utils]
ignore_when_discovered: ["__init__", "hf"]
processors:
- type: filter

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/validators]
search_path: [../haystack/components/validators]
modules: ["json_schema"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -1,6 +1,6 @@
loaders:
- type: haystack_pydoc_tools.loaders.CustomPythonLoader
search_path: [../../../haystack/components/websearch]
search_path: [../haystack/components/websearch]
modules: ["serper_dev", "searchapi"]
ignore_when_discovered: ["__init__"]
processors:

View File

@@ -73,12 +73,14 @@ dependencies = [
# We pin it here to avoid taking too much time.
# https://opendev.org/openstack/reno/src/branch/master/requirements.txt#L7
"dulwich>=0.21.0,<1.0.0",
"haystack-pydoc-tools",
]
[tool.hatch.envs.default.scripts]
release-note = "reno new {args}"
fmt = "ruff check --fix {args} && ruff format {args}"
fmt-check = "ruff check {args} && ruff format --check {args}"
docs = "./.github/utils/pydoc-markdown.sh"
[tool.hatch.envs.test]
@@ -175,14 +177,6 @@ extra-dependencies = [
[tool.hatch.envs.e2e.scripts]
test = "pytest {args:e2e}"
[tool.hatch.envs.readme]
installer = "uv"
detached = true # To avoid installing the dependencies from the default environment
dependencies = ["haystack-pydoc-tools"]
[tool.hatch.envs.readme.scripts]
sync = "./.github/utils/pydoc-markdown.sh {args}"
[project.urls]
"CI: GitHub" = "https://github.com/deepset-ai/haystack/actions"
"Docs: RTD" = "https://haystack.deepset.ai/overview/intro"

View File

@@ -14,7 +14,7 @@ pip install farm-haystack
- Leverage existing **knowledge bases** and better handle the long tail of queries that **chatbots** receive.
- **Automate processes** by automatically applying a list of questions to new documents and using the extracted answers.
![Logo](https://raw.githubusercontent.com/deepset-ai/haystack/main/docs/img/banner.png)
![Logo](https://raw.githubusercontent.com/deepset-ai/haystack/main/images/banner.png)
## Core Features