Makefile: Allow XML documentation to exist outside source files

Moved the xmldoc build logic from the top-level Makefile into
its own script "make_xml_documentation" in the build_tools
directory.

Created a new utility script "get_sourceable_makeopts", also in
the build_tools directory, that dumps the top-level "makeopts"
file in a format that can be "sourced" from shell sscripts.
This allows scripts to easily get the values of common make
build variables such as the location of the GREP, SED, AWK, etc.
utilities as well as the AST* and library *_LIB and *_INCLUDE
variables.

Besides moving logic out of the Makefile, some optimizations
were done like removing "third-party" from the list of
subdirectories to be searched for documentation and changing some
assignments from "=" to ":=" so they're only evaluated once.
The speed increase is noticeable.

The makeopts.in file was updated to include the paths to
REALPATH and DIRNAME.  The ./conifgure script was setting them
but makeopts.in wasn't including them.

So...

With this change, you can now place documentation in any"c"
source file AND you can now place it in a separate XML file
altogether.  The following are examples of valid locations:

res/res_pjsip.c
    Using the existing /*** DOCUMENTATION ***/ fragment.

res/res_pjsip/pjsip_configuration.c
    Using the existing /*** DOCUMENTATION ***/ fragment.

res/res_pjsip/pjsip_doc.xml
    A fully-formed XML file.  The "configInfo", "manager",
    "managerEvent", etc. elements that would be in the "c"
    file DOCUMENTATION fragment should be wrapped in proper
    XML.  Example for "somemodule.xml":

    <?xml version="1.0" encoding="UTF-8"?>
    <!DOCTYPE docs SYSTEM "appdocsxml.dtd">
    <docs>
        <configInfo>
        ...
        </configInfo>
    </docs>

It's the "appdocsxml.dtd" that tells make_xml_documentation
that this is a documentation XML file and not some other XML file.
It also allows many XML-capable editors to do formatting and
validation.

Other than the ".xml" suffix, the name of the file is not
significant.

As a start... This change also moves the documentation that was
in res_pjsip.c to 2 new XML files in res/res_pjsip:
pjsip_config.xml and pjsip_manager.xml.  This cut the number of
lines in res_pjsip.c in half. :)

Change-Id: I486c16c0b5a44d7a8870008e10c941fb19b71ade

Makefile

@@ -475,60 +475,24 @@ endif
 		$(INSTALL) -m 644 $$x "$(DESTDIR)$(ASTDATADIR)/rest-api" ; \
 	done
 
-ifeq ($(GREP),)
-else ifeq ($(GREP),:)
-else
-  XML_core_en_US = $(foreach dir,$(MOD_SUBDIRS),$(shell $(GREP) -l "language=\"en_US\"" $(dir)/*.c $(dir)/*.cc 2>/dev/null))
-endif
-
+DOC_MOD_SUBDIRS := $(filter-out third-party,$(MOD_SUBDIRS))
+XML_core_en_US := $(shell build_tools/make_xml_documentation --command=print_dependencies --source-tree=. --mod-subdirs="$(DOC_MOD_SUBDIRS)")
+# core-en_US.xml is the normal documentation created with asterisk builds.
 doc/core-en_US.xml: makeopts .lastclean $(XML_core_en_US)
-	@printf "Building Documentation For: "
-	@echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" > $@
-	@echo "<!DOCTYPE docs SYSTEM \"appdocsxml.dtd\">" >> $@
-	@echo "<?xml-stylesheet type=\"text/xsl\" href=\"appdocsxml.xslt\"?>" >> $@
-	@echo "<docs xmlns:xi=\"http://www.w3.org/2001/XInclude\">" >> $@
-	@for x in $(MOD_SUBDIRS); do \
-		printf "$$x " ; \
-		for i in `find $$x -name '*.c'`; do \
-			MODULEINFO=$$($(AWK) -f build_tools/get_moduleinfo $$i) ; \
-			if [ -n "$$MODULEINFO" ] ; \
-			then \
-				echo "<module language=\"en_US\" name=\"`$(BASENAME) $$i .c`\">" >> $@ ; \
-				echo "$$MODULEINFO" >> $@ ; \
-				echo "</module>" >> $@ ; \
-			fi ; \
-			$(AWK) -f build_tools/get_documentation $$i >> $@ ; \
-		done ; \
-	done
-	@echo
-	@echo "</docs>" >> $@
+	@build_tools/make_xml_documentation --command=create_xml --source-tree=. --mod-subdirs="$(DOC_MOD_SUBDIRS)" \
+		--with-moduleinfo --validate --output-file=$@
 
-ifeq ($(GREP),)
-else ifeq ($(GREP),:)
-else
-  XML_full_en_US = $(foreach dir,$(MOD_SUBDIRS),$(shell $(GREP) -l "language=\"en_US\"" $(dir)/*.c $(dir)/*.cc 2>/dev/null))
-endif
-
-doc/full-en_US.xml: makeopts .lastclean $(XML_full_en_US)
+# The full-en_US.xml target is only called by the wiki documentation generation process
+# and does special post-processing in preparation for uploading to the wiki.
+# It creates full-en_US.xml but then re-creates core-en_US.xml as well.
+doc/full-en_US.xml: makeopts .lastclean $(XML_core_en_US)
 ifeq ($(PYTHON),:)
 	@echo "--------------------------------------------------------------------------"
 	@echo "---        Please install python to build full documentation           ---"
 	@echo "--------------------------------------------------------------------------"
 else
-	@printf "Building Documentation For: "
-	@echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" > $@
-	@echo "<!DOCTYPE docs SYSTEM \"appdocsxml.dtd\">" >> $@
-	@echo "<?xml-stylesheet type=\"text/xsl\" href=\"appdocsxml.xslt\"?>" >> $@
-	@echo "<docs xmlns:xi=\"http://www.w3.org/2001/XInclude\">" >> $@
-	@for x in $(filter-out third-party,$(MOD_SUBDIRS)); do \
-		printf "$$x " ; \
-		for i in `find $$x -name '*.c'`; do \
-			$(PYTHON) build_tools/get_documentation.py < $$i >> $@ ; \
-		done ; \
-	done
-	@echo
-	@echo "</docs>" >> $@
-	@$(PYTHON) build_tools/post_process_documentation.py -i $@ -o "doc/core-en_US.xml"
+	@build_tools/make_xml_documentation --command=create_xml --source-tree=. --mod-subdirs="$(DOC_MOD_SUBDIRS)" \
+		--for-wiki --validate --output-file=$@ --core-output-file=./doc/core-en_US.xml
 endif
 
 validate-docs: doc/core-en_US.xml

build_tools/get_sourceable_makeopts

+#!/bin/sh
+PROGNAME="${0##*/}"
+
+if [ "$1" = "-h"  ] || [ "$1" = "--help" ] ; then
+	cat <<-EOF
+		Usage: ${PROGNAME}: [ <input_file> ] [ <output_file> ]
+
+		This script takes an Asterisk makeopts file, or any file containing
+		"make" style variable assignments, and converts it into a format
+		that can be directly 'sourced' by shell scripts.
+
+		* Any spaces around the equals sign are removed.
+		* The variable value is quoted.
+		* The "make" "or" command is evaluated.
+
+		Both input and output files are optional and will default to
+		stdin and stdout respectively.
+
+		NOTE: This script relies on NO external commands and only POSIX
+		constructs.  It should be runnable by any shell.
+	EOF
+	exit 1
+fi
+
+input_file="/dev/stdin"
+if [ "$1" != "" ] ; then
+	input_file="$1"
+fi
+
+output_file="/dev/stdout"
+if [ "$2" != "" ] ; then
+	output_file="$2"
+fi
+
+# orfunc is a code fragment to be added to the outp[ut file.
+# We don't WANT the variables evaluated.
+# shellcheck disable=SC2016
+orfunc='or (){ before="${1%,*}" ; after="${1#*,}" ; if [ "$before" = "" ] ; then echo "${after}" ; else echo "${before}" ; fi ; }'
+echo "${orfunc}" >"${output_file}"
+
+while read -r LINE ; do
+	var="${LINE%%=*}"
+	if [ "${var}" != "" ] ; then
+		val="${LINE#*=}"
+		if [ "${val}" != "${var}" ] ; then
+			if [ "${val%% *}" = "" ] ; then
+				echo "${var% *}=\"${val#* }\""
+			else
+				echo "${var% *}=\"${val}\""
+			fi
+		fi
+	fi
+done <"${input_file}"  >>"${output_file}"
+

build_tools/make_xml_documentation

+#!/bin/sh
+# The GREP, SED, FIND, etc variables are all set at run time from
+# makeopts.
+# shellcheck disable=SC2154
+
+PROGNAME="${0##*/}"
+PROGDIR="${0%/*}"
+
+# Fail on errors
+set -e
+
+usage() {
+cat <<-EOF
+	Usage: ${PROGNAME} --command=(create_xml | print_dependencies)
+	--source-tree=<asterisk_source_tree> [ --mod-subdirs=<subdir_search_list> ]
+	[ --with-moduleinfo ] [--for-wiki ] [ --validate ]
+	[ --output-file=<output_xml_file> ]
+	[ --core-output-file=<core_output_xml_file> ]
+
+	command:
+	       print_dependencies:  Print the source files that have documentation
+	                            for use by "make" as dependencies.
+	       create_xml:          Create the actual XML output file.
+
+	source-tree:      The path to the Asterisk source tree.
+
+	mod-subdirs:      A quoted, space-separated list of module sub-directories
+	                  to search for documentation.  Defaults to
+	                  "channels pbx apps codecs formats cdr cel bridges funcs tests main res addons"
+
+	with-moduleinfo:  Include the "MODULEINFO" block from source files.
+	                  Default is to not include MODULEINFO
+
+	for-wiki:         Perform special post processing for wiki documentation.
+	                  This creates two output files and therefore needs both
+	                  <output-file> and <core-output-file>.
+	                  Default is to not perform wiki post-processing.
+
+	validate:         Run xmllint or xmlstarlet to validate output-file.
+
+	output-file:      The XML file to write to if the command was
+	                  "create_xml".
+
+	core-output-file: The additional XML file to write to if the command was
+	                  "create_xml" with "for-wiki".
+
+EOF
+}
+
+with_moduleinfo=0
+for_wiki=0
+validate=0
+command=""
+mod_subdirs="channels pbx apps codecs formats cdr cel bridges funcs tests main res addons"
+source_tree=""
+output_file=""
+core_output_file=""
+
+for arg in "$@" ; do
+	case ${arg} in
+	--for-wiki)
+		for_wiki=1
+		;;
+	--with-moduleinfo)
+		with_moduleinfo=1
+		;;
+	--validate)
+		validate=1
+		;;
+	--command=*)
+		command=${arg#*=}
+		;;
+	--source-tree=*)
+		source_tree=${arg#*=}
+		;;
+	--mod-subdirs=*)
+		mod_subdirs="${arg#*=}"
+		;;
+	--output-file=*)
+		output_file=${arg#*=}
+		;;
+	--core-output-file=*)
+		core_output_file=${arg#*=}
+		;;
+	-h|--help)
+		usage
+		exit 0
+		;;
+	*)
+		echo "unknown option '${arg}'."
+		usage
+		exit 1
+		;;
+	esac
+done
+
+if [ "${command}" = "" ] ; then
+	echo "No command specified"
+	usage
+	exit 1
+fi
+
+if [ "${source_tree}" = "" ] ; then
+	echo "No source-tree specified"
+	usage
+	exit 1;
+fi
+
+if [ ! -d "${source_tree}" ] ; then
+	echo "Asterisk source tree '${source_tree}' doesn't exist."
+	exit 1
+fi
+
+if [ ! -f "${source_tree}/Makefile" ] ; then
+	echo "There's no 'Makefile' in '${source_tree}'."
+	exit 1
+fi
+
+if [ ! -f "${source_tree}/makeopts" ] ; then
+	echo "There's no 'makeopts' in '${source_tree}'.  Maybe you need to run ./configure?"
+	exit 1
+fi
+
+# This will get the paths to the utilities we need, all
+# of which will be in makeopts.  We need to convert the
+# format so it's sourceable.
+tmpname="/tmp/ast_makeopts.$$.env"
+trap 'rm "$tmpname" >/dev/null 2>&1' INT QUIT TERM EXIT
+"${PROGDIR}/get_sourceable_makeopts" "${source_tree}/makeopts" >"${tmpname}"
+# The file to be sourced is generated at run time and can't be checked.
+# shellcheck disable=SC1090
+. "${tmpname}"
+rm "${tmpname}" > /dev/null 2>&1 || :
+trap - INT QUIT TERM EXIT
+
+# Make sure we have everything we need.
+for c in GREP FIND AWK DIRNAME BASENAME SED CAT ; do
+	bin=$(eval "echo \${${c}}")
+	if [ "${bin}" = "" ] ; then
+		echo "The '${c}' utility was not found."
+		exit 1
+	fi
+done
+
+if [ "${for_wiki}" -eq "1" ] || [ "${validate}" -eq "1" ]; then
+	if [ "${XMLLINT}${XMLSTARLET}" = "::" ] ; then
+		echo "Either xmllint or xmlstarlet is required for wiki post-processing or validation."
+		exit 1
+	fi
+fi
+
+if [ "${command}" = "print_dependencies" ] ; then
+	for subdir in ${mod_subdirs} ; do
+		subpath="${source_tree}/${subdir}"
+		# We WANT word splitting in the following line.
+		# shellcheck disable=SC2046
+		${GREP} -l -E '(language="en_US"|appdocsxml.dtd)' $(${FIND} "${subpath}" -name '*.c' -or -name '*.cc' -or -name '*.xml') || :
+	done
+	exit
+fi
+
+if [ "${command}" != "create_xml" ] ; then
+	echo "Command '${command}' is invalid."
+	usage
+	exit 1
+fi
+
+if [ "${output_file}" = "" ] ; then
+	echo "output-file is required for command '${command}'."
+	usage
+	exit 1;
+fi
+
+output_dir=$(${DIRNAME} "${output_file}")
+if [ ! -d "${output_dir}" ] ; then
+	echo "output destination directory '${output_dir}' doesn't exist."
+	exit 1
+fi
+
+if [ "${for_wiki}" -eq "1" ] && [ "${core_output_file}" = "" ] ; then
+	echo "core-output-file is required for command '${command}' and 'for-wiki'."
+	usage
+	exit 1;
+fi
+
+core_output_dir=$(${DIRNAME} "${core_output_file}")
+if [ ! -d "${core_output_dir}" ] ; then
+	echo "core destination directory '${core_output_dir}' doesn't exist."
+	exit 1
+fi
+
+${CAT} > "${output_file}" <<-EOF
+	<?xml version="1.0" encoding="UTF-8"?>
+	<!DOCTYPE docs SYSTEM "appdocsxml.dtd">
+	<?xml-stylesheet type="text/xsl" href="appdocsxml.xslt"?>
+	<docs xmlns:xi="http://www.w3.org/2001/XInclude">
+EOF
+
+printf "Building Documentation For: "
+
+for subdir in ${mod_subdirs} ; do
+	printf "%s " "${subdir}"
+	subdir_path="${source_tree}/${subdir}"
+	for i in $(${FIND} "${subdir_path}" -name '*.c' -or -name '*.cc'); do
+		if [ "${with_moduleinfo}" -eq "1" ] ; then
+			MODULEINFO=$(${AWK} -f "${source_tree}/build_tools/get_moduleinfo" "${i}")
+			if [ "${MODULEINFO}" != "" ] ; then
+				${CAT} >> "${output_file}" <<-EOF
+				<module language="en_US" name="$(${BASENAME} "${i}" .c)">
+				${MODULEINFO}
+				</module>
+				EOF
+			fi
+		fi
+		if [ "${for_wiki}" -eq "1" ] ; then
+			${PYTHON} build_tools/get_documentation.py < "${i}" >> "${output_file}"
+		else
+			${AWK} -f "${source_tree}/build_tools/get_documentation" "${i}" >> "${output_file}"
+		fi
+	done
+	for i in $(${FIND} "${subdir_path}" -name '*.xml') ; do
+		${GREP} -q "appdocsxml.dtd" "${i}" || continue
+		if [ "${validate}" -eq "1" ] ;then
+			if [ "${XMLLINT}" != ":" ] ; then
+				${XMLLINT} --dtdvalid "${source_tree}/doc/appdocsxml.dtd" --path "${source_tree}/doc" --noout "${i}" || { echo "" ; exit 1 ; }
+			else
+				${XMLSTARLET} val -d "${source_tree}/doc/appdocsxml.dtd" "${i}" || { echo "" ; exit 1 ; }
+			fi
+		fi
+		${SED} -r "/^\s*(<[?]xml|<.DOCTYPE|<.?docs)/d" "${i}" >> "${output_file}"
+	done
+done
+echo "</docs>" >> "${output_file}"
+echo ""
+
+if [ "${for_wiki}" -eq "1" ] ; then
+	${PYTHON} build_tools/post_process_documentation.py -i "${output_file}" -o "${core_output_file}"
+fi
+
+if [ "${validate}" -eq "1" ] ;then
+	if [ "${XMLLINT}" != ":" ] ; then
+		${XMLLINT} --dtdvalid "${source_tree}/doc/appdocsxml.dtd" --path "${source_tree}/doc" --noout "${output_file}" || exit 1
+	else
+		${XMLSTARLET} val -d "${source_tree}/doc/appdocsxml.dtd" "${output_file}" || exit 1
+	fi
+fi
+

makeopts.in

@@ -46,6 +46,8 @@ SED=@SED@
 NM=@NM@
 CAT=@CAT@
 CUT=@CUT@
+REALPATH=@REALPATH@
+DIRNAME=@DIRNAME@
 
 BUILD_PLATFORM=@BUILD_PLATFORM@
 BUILD_CPU=@BUILD_CPU@