man: add info on includes to conversion README

doc-migration/README.md

@@ -10,6 +10,7 @@
       - [Custom Sphinx Extensions](#custom-sphinx-extensions)
         - [directive_roles.py (90% done)](#directive_rolespy-90-done)
         - [external_man_links.py](#external_man_linkspy)
+      - [Includes](#includes)
   - [Todo:](#todo)
 
 ## Prerequisites
@@ -119,7 +120,43 @@ This is used to create custom sphinx roles to handle external links for man page
 `:die-net:`refentrytitle(manvolnum)` will lead to 'http://linux.die.net/man/{manvolnum}/{refentrytitle}'
 a full list of these roles can be found in [external_man_links](source/_ext/external_man_links.py).
 
-## Todo:
+#### Includes
+
+1. Versions
+   In the Docbook files you may find lines like these: `<xi:include href="version-info.xml" xpointer="v205"/>` which would render into `Added in version 205` in the docs. This is now archived with the existing [sphinx directive ".. versionadded::"](https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-versionadded) and represented as `.. versionadded:: 205` in the rst file
+
+2. Code Snippets
+   These can be included with the [literalinclude directive](https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-literalinclude) when living in their own file.
+
+   Example:
+
+  ```rst
+  .. literalinclude:: ./check-os-release-simple.py
+  :language: python
+  ```
+
+  There is also the option to include a [code block](https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-code-block) directly in the rst file.
+
+  Example:
+
+  ```rst
+  .. code-block:: sh
+
+  a{sv} 3 One s Eins Two u 2 Yes b true
+
+  ```
+
+3. Text Snippets
+
+  There are a few xml files were sections of these files are reused in multiple other files. While it is no problem to include a whole other rst file the concept of only including a part of that file is a bit more tricky. You can choose to include text partial that starts after a specific text and also to stop before reaching another text. So we decided it would be best to add start and stop markers to define the section in these source files. These markers are: `.. inclusion-marker-do-not-remove` / ``So that a`<xi:include href="standard-options.xml" xpointer="no-pager" />` turns into:
+
+  ```rst
+  .. include:: ./standard-options.rst
+    :start-after: .. inclusion-marker-do-not-remove no-pager
+    :end-before: .. inclusion-end-marker-do-not-remove no-pager
+  ```
+
+## Todo
 
 An incomplete list.
 

doc-migration/convert.sh

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
 # Array of XML filenames
-files=("busctl" "systemd" "journalctl" "os-release")
+files=("sd_journal_get_data" "busctl" "systemd" "journalctl" "os-release")
 
 # Directory paths
 input_dir="../man"

doc-migration/db2rst.py

@@ -315,14 +315,11 @@ def refnamediv(el):
 def refsynopsisdiv(el):
     # return '**Synopsis** \n\n' + _make_title(_join_children(el, ' '), 3)
     s = ""
-    s += '.. only:: html\n\n' + _make_title(_join_children(el, ' — '), 3, 3)
+    s += _make_title('Synopsis', 2, 3)
     s += '\n\n'
-    s += '.. only:: man\n\n' + _make_title('Synopsis', 3, 3)
-    s += '\n\n'
-    s += _join_children(el, ' — ')
+    s += _join_children(el, ', ')
     return s
 
-
 def refname(el):
     _has_only_text(el)
     return "%s" % el.text
@@ -435,8 +432,6 @@ def constant(el):
 
 
 filename = command
-function = command
-
 
 def optional(el):
     return "[%s]" % _concat(el).strip()
@@ -695,6 +690,34 @@ def screen(el):
 def synopsis(el):
     return _indent(el, 3, "::\n\n", False) + "\n\n"
 
+def funcsynopsis(el):
+    return _concat(el)
+
+def funcsynopsisinfo(el):
+    return "``%s``" % _concat(el)
+
+def funcprototype(el):
+    funcdef = ''.join(el.find('.//funcdef').itertext())
+    params = el.findall('.//paramdef')
+    param_list = [''.join(param.itertext()) for param in params]
+    s = ".. code-block:: \n\n   "
+    s += f"{funcdef}("
+    s += ",\n\t".join(param_list)
+    s += ");"
+    return s
+
+def paramdef(el):
+    return el
+
+def funcdef(el):
+    return el
+
+def function(el):
+    return _concat(el).strip()
+
+def parameter(el):
+    return el
+
 
 def userinput(el):
     return _indent(el, 3, "\n\n")

doc-migration/main.py

@@ -3,9 +3,16 @@
 import os
 import json
 import argparse
-from typing import List, Tuple
+from typing import List
 from db2rst import convert_xml_to_rst
 
+FILES_USED_FOR_INCLUDES = [
+    'sd_journal_get_data.xml', 'standard-options.xml', 'user-system-options.xml',
+    'common-variables.xml', 'standard-conf.xml', 'libsystemd-pkgconfig.xml', 'threads-aware.xml'
+]
+
+INCLUDES_DIR = "includes"
+
 
 def load_files_from_json(json_path: str) -> List[str]:
     """
@@ -50,8 +57,12 @@ def process_xml_files_in_directory(dir: str, output_dir: str, specific_file: str
     errored (bool, optional): Flag to process only files listed in errors.json. Defaults to False.
     unhandled_only (bool, optional): Flag to process only files listed in successes_with_unhandled_tags.json. Defaults to False.
     """
-    files_output_dir = os.path.join(output_dir, "")
+    files_output_dir = os.path.join(output_dir, "files")
+    includes_output_dir = os.path.join(output_dir, INCLUDES_DIR)
     os.makedirs(files_output_dir, exist_ok=True)
+    os.makedirs(includes_output_dir, exist_ok=True)
+
+    files_to_process = []
 
     if errored:
         errors_json_path = os.path.join(output_dir, "errors.json")
@@ -66,29 +77,41 @@ def process_xml_files_in_directory(dir: str, output_dir: str, specific_file: str
         if not files_to_process:
             print("No files to process from successes_with_unhandled_tags.json. Exiting.")
             return
-    else:
-        if specific_file:
-            print('hi specific file')
-            specific_file_path = os.path.join(dir, specific_file)
-            if os.path.isfile(specific_file_path):
-                files_to_process = [specific_file]
-            else:
-                print(f"Error: The file '{
-                      specific_file}' does not exist in the directory '{dir}'.")
-                return
+    elif specific_file:
+        specific_file_path = os.path.join(dir, specific_file)
+        if os.path.isfile(specific_file_path):
+            files_to_process = [specific_file]
         else:
-            files_to_process = [f for f in os.listdir(
-                dir) if f.endswith(".xml")]
+            print(f"Error: The file '{
+                  specific_file}' does not exist in the directory '{dir}'.")
+            return
+    else:
+        files_to_process = [f for f in os.listdir(dir) if f.endswith(".xml")]
+
+    errors_json_path = os.path.join(output_dir, "errors.json")
+    unhandled_json_path = os.path.join(
+        output_dir, "successes_with_unhandled_tags.json")
+
+    existing_errors = []
+    existing_unhandled = []
+
+    if os.path.exists(errors_json_path):
+        with open(errors_json_path, 'r') as json_file:
+            existing_errors = json.load(json_file)
+
+    if os.path.exists(unhandled_json_path):
+        with open(unhandled_json_path, 'r') as json_file:
+            existing_unhandled = json.load(json_file)
 
     updated_errors = []
     updated_successes_with_unhandled_tags = []
 
     for filename in files_to_process:
         filepath = os.path.join(dir, filename)
+        output_subdir = includes_output_dir if filename in FILES_USED_FOR_INCLUDES else files_output_dir
         print('converting file: ', filename)
         try:
-            unhandled_tags, error = convert_xml_to_rst(
-                filepath, files_output_dir)
+            unhandled_tags, error = convert_xml_to_rst(filepath, output_subdir)
             if error:
                 result = {
                     "file": filename,
@@ -106,6 +129,12 @@ def process_xml_files_in_directory(dir: str, output_dir: str, specific_file: str
                 }
                 if len(unhandled_tags) > 0:
                     updated_successes_with_unhandled_tags.append(result)
+
+            existing_errors = [
+                entry for entry in existing_errors if entry['file'] != filename]
+            existing_unhandled = [
+                entry for entry in existing_unhandled if entry['file'] != filename]
+
         except Exception as e:
             result = {
                 "file": filename,
@@ -115,27 +144,15 @@ def process_xml_files_in_directory(dir: str, output_dir: str, specific_file: str
             }
             updated_errors.append(result)
 
-    if not specific_file:
-        if not errored:
-            errors_file_path = os.path.join(output_dir, "errors.json")
-            if os.path.exists(errors_file_path):
-                with open(errors_file_path, 'r') as json_file:
-                    existing_errors = json.load(json_file)
-                updated_errors += [
-                    entry for entry in existing_errors if entry['file'] not in files_to_process]
-            update_json_file(errors_file_path, updated_errors)
+    if not errored:
+        updated_errors += existing_errors
 
-        if not unhandled_only:
-            successes_with_unhandled_tags_file_path = os.path.join(
-                output_dir, "successes_with_unhandled_tags.json")
-            if os.path.exists(successes_with_unhandled_tags_file_path):
-                with open(successes_with_unhandled_tags_file_path, 'r') as json_file:
-                    existing_successes_with_unhandled_tags = json.load(
-                        json_file)
-                updated_successes_with_unhandled_tags += [
-                    entry for entry in existing_successes_with_unhandled_tags if entry['file'] not in files_to_process]
-            update_json_file(successes_with_unhandled_tags_file_path,
-                             updated_successes_with_unhandled_tags)
+    if not unhandled_only:
+        updated_successes_with_unhandled_tags += existing_unhandled
+
+    update_json_file(errors_json_path, updated_errors)
+    update_json_file(unhandled_json_path,
+                     updated_successes_with_unhandled_tags)
 
 
 def main():
@@ -144,7 +161,7 @@ def main():
     parser.add_argument(
         "--dir", type=str, help="Path to the directory containing XML files.", default="../man")
     parser.add_argument(
-        "--output", type=str, help="Path to the output directory for results and log files.")
+        "--output", type=str, help="Path to the output directory for results and log files.", default="in-progress")
     parser.add_argument(
         "--file", type=str, help="If provided, the script will only process the specified file.", default=None)
     parser.add_argument("--errored", action='store_true',

doc-migration/source/docs/busctl.rst

@@ -15,15 +15,8 @@ busctl(1)
    busctl — Introspect the bus
    ###########################
 
-.. only:: html
-
-   ``busctl`` [OPTIONS...] [COMMAND] [<NAME>...]
-   =============================================
-
-.. only:: man
-
    Synopsis
-   ========
+   ########
 
 ``busctl`` [OPTIONS...] [COMMAND] [<NAME>...]
 

doc-migration/source/docs/includes/sd_journal_get_data.rst

@@ -0,0 +1,315 @@
+.. SPDX-License-Identifier: LGPL-2.1-or-later:
+
+:title: sd_journal_get_data
+
+:manvolnum: 3
+
+.. _sd_journal_get_data(3):
+
+======================
+sd_journal_get_data(3)
+======================
+
+.. only:: html
+
+   sd_journal_get_data — sd_journal_enumerate_data — sd_journal_enumerate_available_data — sd_journal_restart_data — SD_JOURNAL_FOREACH_DATA — sd_journal_set_data_threshold — sd_journal_get_data_threshold — Read data fields from the current journal entry
+   ###########################################################################################################################################################################################################################################################
+
+   Synopsis
+   ########
+
+``#include <systemd/sd-journal.h>``
+
+.. code-block::
+
+   int sd_journal_get_data(sd_journal *j,
+	const char *field,
+	const void **data,
+	size_t *length);
+
+.. code-block::
+
+   int sd_journal_enumerate_data(sd_journal *j,
+	const void **data,
+	size_t *length);
+
+.. code-block::
+
+   int sd_journal_enumerate_available_data(sd_journal *j,
+	const void **data,
+	size_t *length);
+
+.. code-block::
+
+   void sd_journal_restart_data(sd_journal *j);
+
+.. code-block::
+
+   SD_JOURNAL_FOREACH_DATA(sd_journal *j,
+	const void *data,
+	size_t length);
+
+.. code-block::
+
+   int sd_journal_set_data_threshold(sd_journal *j,
+	size_t sz);
+
+.. code-block::
+
+   int sd_journal_get_data_threshold(sd_journal *j,
+	size_t *sz);
+
+Description
+===========
+
+sd_journal_get_data() gets the data object associated with a specific field
+from the current journal entry. It takes four arguments: the journal context object, a string with the
+field name to request, plus a pair of pointers to pointer/size variables where the data object and its
+size shall be stored in. The field name should be an entry field name. Well-known field names are listed in
+:ref:`systemd.journal-fields(7)`,
+but any field can be specified. The returned data is in a read-only memory map and is only valid until
+the next invocation of sd_journal_get_data(),
+sd_journal_enumerate_data(),
+sd_journal_enumerate_available_data(), or when the read pointer is altered. Note
+that the data returned will be prefixed with the field name and "=". Also note that, by
+default, data fields larger than 64K might get truncated to 64K. This threshold may be changed and turned
+off with sd_journal_set_data_threshold() (see below).
+
+sd_journal_enumerate_data() may be used
+to iterate through all fields of the current entry. On each
+invocation the data for the next field is returned. The order of
+these fields is not defined. The data returned is in the same
+format as with sd_journal_get_data() and also
+follows the same life-time semantics.
+
+sd_journal_enumerate_available_data() is similar to
+sd_journal_enumerate_data(), but silently skips any fields which may be valid, but
+are too large or not supported by current implementation.
+
+sd_journal_restart_data() resets the
+data enumeration index to the beginning of the entry. The next
+invocation of sd_journal_enumerate_data()
+will return the first field of the entry again.
+
+Note that the SD_JOURNAL_FOREACH_DATA() macro may be used as a handy wrapper
+around sd_journal_restart_data() and
+sd_journal_enumerate_available_data().
+
+Note that these functions will not work before
+:ref:`sd_journal_next(3)`
+(or related call) has been called at least once, in order to
+position the read pointer at a valid entry.
+
+sd_journal_set_data_threshold() may be
+used to change the data field size threshold for data returned by
+sd_journal_get_data(),
+sd_journal_enumerate_data() and
+sd_journal_enumerate_unique(). This threshold
+is a hint only: it indicates that the client program is interested
+only in the initial parts of the data fields, up to the threshold
+in size — but the library might still return larger data objects.
+That means applications should not rely exclusively on this
+setting to limit the size of the data fields returned, but need to
+apply an explicit size limit on the returned data as well. This
+threshold defaults to 64K by default. To retrieve the complete
+data fields this threshold should be turned off by setting it to
+0, so that the library always returns the complete data objects.
+It is recommended to set this threshold as low as possible since
+this relieves the library from having to decompress large
+compressed data objects in full.
+
+sd_journal_get_data_threshold() returns
+the currently configured data field size threshold.
+
+Return Value
+============
+
+sd_journal_get_data() returns 0 on success or a negative errno-style error
+code. sd_journal_enumerate_data() and
+sd_journal_enumerate_available_data() return a positive integer if the next field
+has been read, 0 when no more fields remain, or a negative errno-style error code.
+sd_journal_restart_data() doesn't return anything.
+sd_journal_set_data_threshold() and sd_journal_get_threshold()
+return 0 on success or a negative errno-style error code.
+
+Errors
+------
+
+Returned errors may indicate the following problems:
+
+.. inclusion-marker-do-not-remove EINVAL
+
+.. option:: -EINVAL
+
+   One of the required parameters is ``NULL`` or invalid.
+
+   .. only:: html
+
+      .. versionadded:: 246
+
+.. inclusion-end-marker-do-not-remove EINVAL
+
+.. inclusion-marker-do-not-remove ECHILD
+
+.. option:: -ECHILD
+
+   The journal object was created in a different process, library or module instance.
+
+   .. only:: html
+
+      .. versionadded:: 246
+
+.. inclusion-end-marker-do-not-remove ECHILD
+
+.. inclusion-marker-do-not-remove EADDRNOTAVAIL
+
+.. option:: -EADDRNOTAVAIL
+
+   The read pointer is not positioned at a valid entry;
+   :ref:`sd_journal_next(3)`
+   or a related call has not been called at least once.
+
+   .. only:: html
+
+      .. versionadded:: 246
+
+.. inclusion-end-marker-do-not-remove EADDRNOTAVAIL
+
+.. inclusion-marker-do-not-remove ENOENT
+
+.. option:: -ENOENT
+
+   The current entry does not include the specified field.
+
+   .. only:: html
+
+      .. versionadded:: 246
+
+.. inclusion-end-marker-do-not-remove ENOENT
+
+.. inclusion-marker-do-not-remove ENOMEM
+
+.. option:: -ENOMEM
+
+   Memory allocation failed.
+
+   .. only:: html
+
+      .. versionadded:: 246
+
+.. inclusion-end-marker-do-not-remove ENOMEM
+
+.. inclusion-marker-do-not-remove ENOBUFS
+
+.. option:: -ENOBUFS
+
+   A compressed entry is too large.
+
+   .. only:: html
+
+      .. versionadded:: 246
+
+.. inclusion-end-marker-do-not-remove ENOBUFS
+
+.. inclusion-marker-do-not-remove E2BIG
+
+.. option:: -E2BIG
+
+   The data field is too large for this computer architecture (e.g. above 4 GB on a
+   32-bit architecture).
+
+   .. only:: html
+
+      .. versionadded:: 246
+
+.. inclusion-end-marker-do-not-remove E2BIG
+
+.. inclusion-marker-do-not-remove EPROTONOSUPPORT
+
+.. option:: -EPROTONOSUPPORT
+
+   The journal is compressed with an unsupported method or the journal uses an
+   unsupported feature.
+
+   .. only:: html
+
+      .. versionadded:: 246
+
+.. inclusion-end-marker-do-not-remove EPROTONOSUPPORT
+
+.. inclusion-marker-do-not-remove EBADMSG
+
+.. option:: -EBADMSG
+
+   The journal is corrupted (possibly just the entry being iterated over).
+
+   .. only:: html
+
+      .. versionadded:: 246
+
+.. inclusion-end-marker-do-not-remove EBADMSG
+
+.. inclusion-marker-do-not-remove EIO
+
+.. option:: -EIO
+
+   An I/O error was reported by the kernel.
+
+   .. only:: html
+
+      .. versionadded:: 246
+
+.. inclusion-end-marker-do-not-remove EIO
+
+Notes
+=====
+
+.. include:: ../includes/threads-aware.rst
+                    :start-after: .. inclusion-marker-do-not-remove strict
+                    :end-before: .. inclusion-end-marker-do-not-remove strict
+
+.. include:: ../includes/libsystemd-pkgconfig.rst
+                    :start-after: .. inclusion-marker-do-not-remove pkgconfig-text
+                    :end-before: .. inclusion-end-marker-do-not-remove pkgconfig-text
+
+Examples
+========
+
+See
+:ref:`sd_journal_next(3)`
+for a complete example how to use
+sd_journal_get_data().
+
+Use the
+SD_JOURNAL_FOREACH_DATA() macro to
+iterate through all fields of the current journal
+entry:
+
+.. code-block:: sh
+
+   …
+   int print_fields(sd_journal *j) {
+     const void *data;
+     size_t length;
+     SD_JOURNAL_FOREACH_DATA(j, data, length)
+       printf("%.*s\n", (int) length, data);
+   }
+   …
+
+History
+=======
+
+sd_journal_get_data(),
+sd_journal_enumerate_data(),
+sd_journal_restart_data(), and
+SD_JOURNAL_FOREACH_DATA() were added in version 187.
+
+sd_journal_set_data_threshold() and
+sd_journal_get_data_threshold() were added in version 196.
+
+sd_journal_enumerate_available_data() was added in version 246.
+
+See Also
+========
+
+:ref:`systemd(1)`, :ref:`systemd.journal-fields(7)`, :ref:`sd-journal(3)`, :ref:`sd_journal_open(3)`, :ref:`sd_journal_next(3)`, :ref:`sd_journal_get_realtime_usec(3)`, :ref:`sd_journal_query_unique(3)`

doc-migration/source/docs/journalctl.rst

@@ -15,15 +15,8 @@ journalctl(1)
    journalctl — Print log entries from the systemd journal
    #######################################################
 
-.. only:: html
-
-   ``journalctl`` [OPTIONS...] [MATCHES...]
-   ========================================
-
-.. only:: man
-
    Synopsis
-   ========
+   ########
 
 ``journalctl`` [OPTIONS...] [MATCHES...]
 

doc-migration/source/docs/os-release.rst

@@ -15,15 +15,8 @@ os-release(5)
    os-release — initrd-release — extension-release — Operating system identification
    #################################################################################
 
-.. only:: html
-
-   ``/etc/os-release`` ``/usr/lib/os-release`` ``/etc/initrd-release`` ``/usr/lib/extension-release.d/extension-release.<IMAGE>``
-   ==============================================================================================================================
-
-.. only:: man
-
    Synopsis
-   ========
+   ########
 
 ``/etc/os-release``
 ``/usr/lib/os-release``
@@ -566,7 +559,7 @@ Reading ``os-release`` in :die-net:`python(1)` (versions >= 3.10)
 .. literalinclude:: /code-examples/py/check-os-release-simple.py
                     :language: python
 
-See docs for ```platform.freedesktop_os_release`` <https://docs.python.org/3/library/platform.html#platform.freedesktop_os_release>`_ for more details.
+See docs for `platform.freedesktop_os_release <https://docs.python.org/3/library/platform.html#platform.freedesktop_os_release>`_ for more details.
 
 Reading ``os-release`` in :die-net:`python(1)` (any version)
 ------------------------------------------------------------

doc-migration/source/docs/systemd.rst

@@ -15,17 +15,10 @@ systemd(1)
    systemd — init — systemd system and service manager
    ###################################################
 
-.. only:: html
-
-   ``/usr/lib/systemd/systemd`` [OPTIONS...] — ``init`` [OPTIONS...] {COMMAND}
-   ===========================================================================
-
-.. only:: man
-
    Synopsis
-   ========
+   ########
 
-``/usr/lib/systemd/systemd`` [OPTIONS...] — ``init`` [OPTIONS...] {COMMAND}
+``/usr/lib/systemd/systemd`` [OPTIONS...], ``init`` [OPTIONS...] {COMMAND}
 
 Description
 ===========
@@ -330,7 +323,7 @@ The service listens to various UNIX process signals that can be used to request
 asynchronously. The signal handling is enabled very early during boot, before any further processes are
 invoked. However, a supervising container manager or similar that intends to request these operations via
 this mechanism must take into consideration that this functionality is not available during the earliest
-initialization phase. An ``sd_notify()`` notification message carrying the
+initialization phase. An sd_notify() notification message carrying the
 ``X_SYSTEMD_SIGNALS_LEVEL=2`` field is emitted once the signal handlers are enabled, see
 below. This may be used to schedule submission of these signals correctly.
 

doc-migration/source/index.rst

@@ -27,6 +27,7 @@ systemd — System and Service Manager
    docs/os-release
    docs/systemd
    docs/systemD-directives
+   docs/includes/sd_journal_get_data
 
 Indices and tables
 ==================