From be59bc62efcae7babdb23135bb1bd6e89be1ec93 Mon Sep 17 00:00:00 2001 From: Evan Lenz Date: Thu, 1 Apr 2021 02:03:43 -0700 Subject: [PATCH 1/2] Add crappy starter bash scripts for awesome visualization build relates to #66 --- .gitmodules | 3 + example_visualized/.gitignore | 10 ++ example_visualized/Jamfile | 84 +++++++++++++ example_visualized/README.md | 117 ++++++++++++++++++ example_visualized/assemble-quickbook.sh | 8 ++ example_visualized/clean-build.sh | 11 ++ example_visualized/execute-stage1.sh | 4 + example_visualized/execute-stage2.sh | 7 ++ example_visualized/execute-stages.sh | 2 + example_visualized/extract-xml-pages.sh | 8 ++ .../get-last-built-doxygen-output.sh | 12 ++ example_visualized/get-qbk-pages.sh | 17 +++ example_visualized/non-traced-build.sh | 29 +++++ example_visualized/prepare-stage1.sh | 2 + example_visualized/prepare-stage2.sh | 2 + example_visualized/prepare.sh | 2 + example_visualized/render-stage1.sh | 16 +++ example_visualized/render-stage2.sh | 12 ++ example_visualized/render.sh | 2 + example_visualized/run-all.sh | 63 ++++++++++ example_visualized/stage1-trace-only.sh | 39 ++++++ example_visualized/stage2-params.xml | 3 + example_visualized/trace-stage1.sh | 6 + example_visualized/trace-stage2.sh | 11 ++ example_visualized/trace.sh | 2 + xslt-visualizer | 1 + 26 files changed, 473 insertions(+) create mode 100644 .gitmodules create mode 100644 example_visualized/.gitignore create mode 100644 example_visualized/Jamfile create mode 100644 example_visualized/README.md create mode 100755 example_visualized/assemble-quickbook.sh create mode 100755 example_visualized/clean-build.sh create mode 100755 example_visualized/execute-stage1.sh create mode 100755 example_visualized/execute-stage2.sh create mode 100755 example_visualized/execute-stages.sh create mode 100755 example_visualized/extract-xml-pages.sh create mode 100755 example_visualized/get-last-built-doxygen-output.sh create mode 100755 example_visualized/get-qbk-pages.sh create mode 100755 example_visualized/non-traced-build.sh create mode 100755 example_visualized/prepare-stage1.sh create mode 100755 example_visualized/prepare-stage2.sh create mode 100755 example_visualized/prepare.sh create mode 100755 example_visualized/render-stage1.sh create mode 100755 example_visualized/render-stage2.sh create mode 100755 example_visualized/render.sh create mode 100755 example_visualized/run-all.sh create mode 100755 example_visualized/stage1-trace-only.sh create mode 100644 example_visualized/stage2-params.xml create mode 100755 example_visualized/trace-stage1.sh create mode 100755 example_visualized/trace-stage2.sh create mode 100755 example_visualized/trace.sh create mode 160000 xslt-visualizer diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..ccaaa34 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "xslt-visualizer"] + path = xslt-visualizer + url = git@github.com:evanlenz/xslt-visualizer.git diff --git a/example_visualized/.gitignore b/example_visualized/.gitignore new file mode 100644 index 0000000..8c220a8 --- /dev/null +++ b/example_visualized/.gitignore @@ -0,0 +1,10 @@ +build +html +qbk +images +index.xml +custom-index.xml +main.qbk +reference.qbk +xml-pages.xml +boostbook.dtd diff --git a/example_visualized/Jamfile b/example_visualized/Jamfile new file mode 100644 index 0000000..33582ca --- /dev/null +++ b/example_visualized/Jamfile @@ -0,0 +1,84 @@ +# +# Copyright (c) 2019 Vinnie Falco (vinnie.falco@gmail.com) +# +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# +# Official repository: https://github.com/cppalliance/json +# + +project docca/example_visualized ; + +import os ; +import path ; +import boostbook ; +import quickbook ; +import xsltproc ; +import doxygen ; +import modules ; +import saxonhe ; + + +# We have to make a copy of reference.qbk and put it +# in a place where the static .qbk files can find it +# +install qbk : reference.qbk ; + +#------------------------------------------------------------------------------- +# +# Produce the Boost.Book XML from the QuickBook +# + +install images + : + [ glob images/*.png ] + : + html/json/images + ; + +explicit images ; + +xml json_doc + : + main.qbk + : + images + qbk + ; + +explicit json_doc ; + +#------------------------------------------------------------------------------- +# +# HTML documentation for $(BOOST_ROOT)/doc/html +# +#------------------------------------------------------------------------------- + +boostbook json + : + json_doc + : + boost.root=../../../.. + chapter.autolabel=1 + chunk.section.depth=8 # Depth to which sections should be chunked + chunk.first.sections=1 # Chunk the first top-level section? + toc.section.depth=2 # How deep should recursive sections appear in the TOC? + toc.max.depth=8 # How many levels should be created for each TOC? + generate.section.toc.level=8 # Control depth of TOC generation in sections + generate.toc="chapter toc,title section nop reference nop" + ../../../tools/boostbook/dtd + : + images + ; + +#------------------------------------------------------------------------------- +# +# These are used to inform the build system of the +# means to build the integrated and stand-alone docs. +# + +alias boostdoc ; +explicit boostdoc ; + +alias boostrelease : json ; +explicit boostrelease ; diff --git a/example_visualized/README.md b/example_visualized/README.md new file mode 100644 index 0000000..0b0d77c --- /dev/null +++ b/example_visualized/README.md @@ -0,0 +1,117 @@ +The bash scripts in this directory are here for two main purposes: + + 1. generating the XSLT visualizations for both stages of page rendering (from XML to XML to QuickBook), and + 2. supporting partial builds (to enable faster iterations for development). + +# Architectural Overview +The Boost.Build-based Jamfile in the regular "example" build is sufficient to build all of the HTML +docs using the new XSLT code. It has several phases of processing, each using Saxon-HE: + + 1. **extract-xml-pages.xsl**: + Generate XML page source files, discovered using Doxygen's index.xml. + This yields a subset of Doxygen's output for each page, gathering together just the info + needed for that page (along with some annotations like link refids), but otherwise leaving its + Doxygen-based format unchanged. + 2. **stage1.xsl**: + Transform each Doxygen-style XML page into a simpler XML page format tailored towards the + structure and features of the HTML page it will eventually produce. + 3. **stage2.xsl**: + Transform each tailored XML page into QuickBook format. + 4. **assemble-quickbook.xsl**: + Gather together all the individual Quickbook-formatted results into a single reference.qbk file, + which is used by the rest of the build to weave them into the QuickBook->BoostBook->DocBook->HTML process. + +These same steps are used in both the "example" and "example_visualized" builds. The difference is that the +latter also includes visualizations, and it uses bash shell scripts to drive the whole thing, including +the part that still depends on Boost.Build (the QuickBook->BoostBook->DocBook->HTML process, defined in +example_visualized/Jamfile). + +For a summary of what transient files are used in the example_visualized build (including ones you edit to +configure a partial build), see the .gitignore file in this directory, which also includes comments +explaining each entry. + +# Setup (two scripts) +The initial setup depends on the "example" build, first to get the XML files generated by Doxygen, +and second, to simply copy the original qbk and images directories. These steps are independent +of each other, but both of them must be run. + +## Getting the Doxygen output +For initial setup, after running a full "example" build (i.e. in boost/tools/docca/example), run the +following script: + + * ./get-last-built-doxygen-output.sh + +It will blow away the example_visualized/build directory (if it exists) and create a new one, +populated with all the XML files output by Doxygen, including index.xml. Additionally, it will +copy index.xml to example_visualized/custom-index.xml (which you can customize for partial builds). +If the script fails, you probably will need to modify it, specifically the hard-coded path of +the build dir used by Boost.Build for the regular "example" build. + +[FIXME: Don't hard-code the build directory for the "example" build...] + +## Copying the manually-authored QuickBook +You also need a copy of the qbk and images directories. Run this script to get those: + + * ./get-qbk-pages.sh + + +# Running a build (one master script) +Once you've successfully run the above setup scripts, you can now run a full build of all of the pages, +including both the HTML results and the visualizations, by executing this command: + + * ./run-all.sh + +The HTML results will appear in the "html" directory, and the XSLT visualizations will appear in +subdirectories of the build directory. To find them, understand that a DEBUG option is set (enabled +in the relevant script, trace.sh, which is automatically invoked by run-all.sh) and which causes +every HTML page in the result to include links (immediately following the page's "brief description") +to both of the XSLT visualizations for that page. When reviewing an HTML page in the browser, look +for the links called "stage1_visualized" and "stage2_visualized". Clicking these will yield the +relevant visualizations for that page. You can use this to better understand how the two stages of +XSLT are used to generate the QuickBook, which can also be helpful for debugging. + +NOTE: The stage-two visualizations have become the most time-consuming to generate. For most use cases, +I suspect that ./non-traced-build.sh or ./stage1-trace-only.sh is what you will want to invoke instead of +./run-all.sh (see the sections about this below). + +## Running a partial build (editing custom-index.xml) +If you want to only do a partial build, edit the custom-index.xml in the example_visualized directory +and then comment out everything but the pages you want to include in the build. The +extract-xml-pages.sh script (invoked automatically by run-all.sh) will overwrite build/index.xml with your +modified custom-index.xml, using it as input instead. (For that reason, you may want to comment things out so you +can easily uncomment them later, without having to retrieve the Doxygen output again.) After you've created +your pared-down custom-index.xml, simply run run-all.sh again to generate only those pages. + +## Including previously-built pages in the HTML output +The aforementioned extract-xml-pages.sh script generates all of the input XML "page" files that are +used as input to stage 1 of the page rendering. It also generates a hierarchical list of pages +in build/xml-pages.xml. This, in turn, is used by assemble-quickbook.sh (automatically invoked by run-all.sh), +to pull all of the reference pages (rendered as QuickBook) together into reference.qbk. If you want to +include previously-built reference pages into reference.qbk even after switching to a different partial +build (by editing your custom-index.xml), you can do this by also supplying a custom xml-pages.xml file +in the example_visualized directory (e.g. the result of a full build from earlier, at build/xml-pages.xml). +The assemble-quickbook.sh script will overwrite build/xml-pages.xml with your modified xml-pages.xml, +using it as input instead. (Unlike build/index.xml, build/xml-pages.xml gets regenerated every time you run run-all.sh.) +The purpose of doing this would be, for example, to always get a full build of the HTML docs (for example, +to test links between all the pages) without having to rebuild the QuickBook (and XSLT visualizations) +for all the reference pages every time. + +However, if you want the fastest build possible (to repeatedly render just a subset of the reference pages), +you're best off not using a custom xml-pages.xml file. Just whittling down custom-index.xml to a small subset will +give you the fastest build. + +# Running a build without visualizations (one master script) +As a way of speeding things up further, you can run a build that includes everything but the visualizations +(which is the most time-consuming thing to generate). To do this, simply run: + + * ./non-traced-build.sh + +in lieu of run-all.sh. The technique for doing partial builds still applies to this build script as well. + +# Running a build with visualizations for stage one only (one master script) +As a compromise, you can run a build that includes everything but the stage-two visualization +(which is the most time-consuming thing to generate). To do this, simply run: + + * ./stage1-trace-only.sh + +in lieu of run-all.sh. The technique for doing partial builds still applies to this build script as well. diff --git a/example_visualized/assemble-quickbook.sh b/example_visualized/assemble-quickbook.sh new file mode 100755 index 0000000..92d517a --- /dev/null +++ b/example_visualized/assemble-quickbook.sh @@ -0,0 +1,8 @@ +echo "Copying a manually-modified xml-pages.xml if present..." +cp ../xml-pages.xml . + +echo "Assembling the stage2 results into reference.qbk" +java -cp "$CLASSPATH" net.sf.saxon.Transform -s:xml-pages.xml -xsl:assemble-quickbook.xsl -o:reference.qbk input-dir=stage2_visualized/results + +echo "Copying reference.qbk..." +cp reference.qbk .. diff --git a/example_visualized/clean-build.sh b/example_visualized/clean-build.sh new file mode 100755 index 0000000..11358ff --- /dev/null +++ b/example_visualized/clean-build.sh @@ -0,0 +1,11 @@ +echo "Wiping out previous visualizer build results..." +cat .gitignore | xargs rm -r + +echo "Getting last-built Doxygen output..." +./get-last-built-doxygen-output.sh && + +echo "Getting qbk pages..." +./get-qbk-pages.sh && + +./run-all.sh + diff --git a/example_visualized/execute-stage1.sh b/example_visualized/execute-stage1.sh new file mode 100755 index 0000000..b210256 --- /dev/null +++ b/example_visualized/execute-stage1.sh @@ -0,0 +1,4 @@ +mkdir -p stage1_visualized/results + +echo "Executing stage1..." +java -cp "$CLASSPATH" net.sf.saxon.Transform -threads:128 -s:xml-pages -o:stage1_visualized/results -xsl:stage1.xsl diff --git a/example_visualized/execute-stage2.sh b/example_visualized/execute-stage2.sh new file mode 100755 index 0000000..c9c7ef0 --- /dev/null +++ b/example_visualized/execute-stage2.sh @@ -0,0 +1,7 @@ +mkdir -p stage2 + +echo "Executing stage2..." +java -cp "$CLASSPATH" net.sf.saxon.Transform -threads:128 -s:stage1_visualized/results -o:stage2_visualized/results -xsl:stage2.xsl DEBUG=yes + +echo "Copying the stage2 results to .txt files (for easier viewing in the browser)..." +java -cp "$CLASSPATH" net.sf.saxon.Transform -s:xml-pages.xml -xsl:debug-friendly-quickbook.xsl input-dir=stage2_visualized/results >/dev/null diff --git a/example_visualized/execute-stages.sh b/example_visualized/execute-stages.sh new file mode 100755 index 0000000..a5f7bf7 --- /dev/null +++ b/example_visualized/execute-stages.sh @@ -0,0 +1,2 @@ +./execute-stage1.sh +./execute-stage2.sh diff --git a/example_visualized/extract-xml-pages.sh b/example_visualized/extract-xml-pages.sh new file mode 100755 index 0000000..7455f7f --- /dev/null +++ b/example_visualized/extract-xml-pages.sh @@ -0,0 +1,8 @@ +echo "Copying a custom, manually-slimmed-down index (custom-index.xml), if present..." +cp ../custom-index.xml index.xml + +echo "Removing the previously-extracted XML pages (xml-pages directory)..." +rm -rf xml-pages + +echo "Extracting the xml-pages from index.xml and accompanying files..." +java -cp "$CLASSPATH" net.sf.saxon.Transform -s:index.xml -xsl:extract-xml-pages.xsl -o:xml-pages.xml diff --git a/example_visualized/get-last-built-doxygen-output.sh b/example_visualized/get-last-built-doxygen-output.sh new file mode 100755 index 0000000..061ff20 --- /dev/null +++ b/example_visualized/get-last-built-doxygen-output.sh @@ -0,0 +1,12 @@ +echo "Removing a previous build dir, if it exists..." +rm -rf build +mkdir build + +echo "Attempting to copy the Doxygen output from the last 'example' build; you likely will need to modify my hard-coded source directory..." +cp ../../../bin.v2/tools/docca/example/*.xml build + +echo "Copying custom-overrides.xsl from the build directory..." +cp ../../../bin.v2/tools/docca/example/custom-overrides.xsl build + +echo "Adding a customizable copy of index.xml to example_visualized/custom-index.xml..." +cp build/index.xml custom-index.xml diff --git a/example_visualized/get-qbk-pages.sh b/example_visualized/get-qbk-pages.sh new file mode 100755 index 0000000..04908b5 --- /dev/null +++ b/example_visualized/get-qbk-pages.sh @@ -0,0 +1,17 @@ +echo "Deleting any previous qbk and images directories..." +rm -rf qbk images + +echo "Copying main.qbk from ../example..." +cp -r ../example/main.qbk . + +echo "Copying qbk from ../example/qbk..." +cp -r ../example/qbk . + +echo "Copying images (if any) from ../example/images" +cp -r ../example/images . 2>/dev/null + +echo "Copying boostbook.dtd into the example_visualized dir..." +cp ../example/boostbook.dtd . + +echo "Copying ../example/index.xml into the example_visualized dir..." +cp ../example/index.xml . diff --git a/example_visualized/non-traced-build.sh b/example_visualized/non-traced-build.sh new file mode 100755 index 0000000..9095d1a --- /dev/null +++ b/example_visualized/non-traced-build.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +date + +echo "Copying docca XSLT files..." && \ +cp ../include/docca/* build && \ + +echo "Copying shell scripts..." && \ +cp extract-xml-pages.sh \ + assemble-quickbook.sh \ + execute-stages.sh execute-stage1.sh execute-stage2.sh \ +build && \ + +cd build && \ + +echo "Calling extract-xml-pages.sh..." && \ +./extract-xml-pages.sh && \ + +echo "Running execute-stages.sh..." && \ +./execute-stages.sh && \ + +echo "Calling assemble-quickbook.sh..." && \ +./assemble-quickbook.sh && \ + +echo "Calling the example build to run the Quickbook -> BoostBook -> DocBook -> HTML conversion..." && \ +cd .. && \ +../../../b2.exe + +date diff --git a/example_visualized/prepare-stage1.sh b/example_visualized/prepare-stage1.sh new file mode 100755 index 0000000..cf3bffd --- /dev/null +++ b/example_visualized/prepare-stage1.sh @@ -0,0 +1,2 @@ +rm -rf stage1_visualized/code-trace-enabled +java -cp "$CLASSPATH" net.sf.saxon.Transform -s:stage1.xsl -o:stage1_visualized/code-trace-enabled/stage1.xsl -xsl:xslt-visualizer/xsl/trace-enable.xsl diff --git a/example_visualized/prepare-stage2.sh b/example_visualized/prepare-stage2.sh new file mode 100755 index 0000000..1368197 --- /dev/null +++ b/example_visualized/prepare-stage2.sh @@ -0,0 +1,2 @@ +rm -rf stage2_visualized/code-trace-enabled +java -cp "$CLASSPATH" net.sf.saxon.Transform -s:stage2.xsl -o:stage2_visualized/code-trace-enabled/stage2.xsl -xsl:xslt-visualizer/xsl/trace-enable.xsl diff --git a/example_visualized/prepare.sh b/example_visualized/prepare.sh new file mode 100755 index 0000000..5d11bc0 --- /dev/null +++ b/example_visualized/prepare.sh @@ -0,0 +1,2 @@ +./prepare-stage1.sh +./prepare-stage2.sh diff --git a/example_visualized/render-stage1.sh b/example_visualized/render-stage1.sh new file mode 100755 index 0000000..2d72149 --- /dev/null +++ b/example_visualized/render-stage1.sh @@ -0,0 +1,16 @@ +mkdir -p stage1_visualized/visualized/assets +cp -ru xslt-visualizer/assets/* stage1_visualized/visualized/assets + + +echo "Rendering the stage1 visualization results" +# Create the rendered results (Saxon forces .xml prefix in the output files) +java -cp "$CLASSPATH" net.sf.saxon.Transform -threads:128 -s:stage1_visualized/results/trace-data -o:stage1_visualized/visualized -xsl:xslt-visualizer/xsl/render.xsl + +echo "Renaming the stage1 visualization files to .html" +cd stage1_visualized/visualized +for file in *.xml +do + mv "$file" "${file%.xml}.html" +done + +cd ../.. diff --git a/example_visualized/render-stage2.sh b/example_visualized/render-stage2.sh new file mode 100755 index 0000000..999122e --- /dev/null +++ b/example_visualized/render-stage2.sh @@ -0,0 +1,12 @@ +mkdir -p stage2_visualized/visualized/assets +cp -ru xslt-visualizer/assets/* stage2_visualized/visualized/assets + +echo "Rendering the stage2 visualization results" +java -cp "$CLASSPATH" net.sf.saxon.Transform -threads:128 -s:stage2_visualized/results/trace-data -o:stage2_visualized/visualized -xsl:xslt-visualizer/xsl/render.xsl + +echo "Renaming the stage2 visualization files to .html" +cd stage2_visualized/visualized +for file in *.xml +do + mv "$file" "${file%.xml}.html" +done diff --git a/example_visualized/render.sh b/example_visualized/render.sh new file mode 100755 index 0000000..0c86023 --- /dev/null +++ b/example_visualized/render.sh @@ -0,0 +1,2 @@ +./render-stage1.sh +./render-stage2.sh diff --git a/example_visualized/run-all.sh b/example_visualized/run-all.sh new file mode 100755 index 0000000..55ebe29 --- /dev/null +++ b/example_visualized/run-all.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +date + +echo "Copying docca XSLT files..." && \ +cp ../include/docca/* build && \ + +echo "Copying xslt-visualizer..." && \ +rm -rf build/xslt-visualizer && \ +cp -r ../xslt-visualizer build && \ + +echo "Copying shell scripts..." && \ +cp extract-xml-pages.sh \ + prepare.sh prepare-stage1.sh prepare-stage2.sh \ + trace.sh trace-stage1.sh trace-stage2.sh \ + render.sh render-stage1.sh render-stage2.sh \ + assemble-quickbook.sh \ +build && \ + +echo "Copying stage2 parameters file..." && \ +cp stage2-params.xml build && \ + +cd build && \ + +echo "Calling extract-xml-pages.sh..." && \ +./extract-xml-pages.sh && \ + +echo "Calling prepare.sh..." && \ +./prepare.sh && \ + +echo "Calling trace.sh..." && \ +./trace.sh && \ + +echo "Calling render.sh..." && \ +./render.sh && \ + +echo "Calling assemble-quickbook.sh..." && \ +./assemble-quickbook.sh && \ + +echo "Calling the example build to run the Quickbook -> BoostBook -> DocBook -> HTML conversion..." && \ +cd .. && \ +../../../b2.exe + +date + + + + +# Everything below tries to replicate what the b2.exe call does above +#echo "Copying reference.qbk into qbk..." && \ +#cp ../reference.qbk ../qbk && \ + +#echo "Converting QuickBook (main.qbk) to BoostBook (beast_doc.xml)..." && \ +#../../../../bin.v2/tools/quickbook/src/msvc-14.2/release/cxxstd-0x-iso/link-static/threading-multi/quickbook.exe --output-file=beast_doc.xml ../qbk/main.qbk && \ + +#echo "Converting BoostBook (beast_doc.xml) to DocBook (beast_doc.docbook)..." && \ +#set XML_CATALOG_FILES=../../../../bin.v2/boostbook_catalog.xml && \ +#/usr/bin/xsltproc --stringparam boost.defaults "Boost" --stringparam boost.root "../../../.." --stringparam chapter.autolabel "1" --stringparam chunk.first.sections "1" --stringparam chunk.section.depth "8" --stringparam generate.section.toc.level "8" --stringparam generate.toc "chapter toc,title section nop reference nop" --stringparam toc.max.depth "8" --stringparam toc.section.depth "8" --path "../../../../bin.v2" --xinclude -o beast_doc.docbook ../../../../tools/boostbook/xsl/docbook.xsl beast_doc.xml && \ + + +#echo "Converting DocBook (beast_doc.docbook) to HTML..." && \ +#set XML_CATALOG_FILES=../../../../bin.v2/boostbook_catalog.xml && \ +#/usr/bin/xsltproc --stringparam boost.defaults "Boost" --stringparam boost.root "../../../.." --stringparam chapter.autolabel "1" --stringparam chunk.first.sections "1" --stringparam chunk.section.depth "8" --stringparam generate.section.toc.level "8" --stringparam generate.toc "chapter toc,title section nop reference nop" --stringparam manifest "beast_HTML.manifest" --stringparam toc.max.depth "8" --stringparam toc.section.depth "8" --path "../../../../bin.v2" --xinclude -o ../html/ ../../../../tools/boostbook/xsl/html.xsl beast_doc.docbook diff --git a/example_visualized/stage1-trace-only.sh b/example_visualized/stage1-trace-only.sh new file mode 100755 index 0000000..b8b115e --- /dev/null +++ b/example_visualized/stage1-trace-only.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +date + +echo "Copying docca XSLT files..." && \ +cp ../include/docca/* build && \ + +echo "Copying shell scripts..." && \ +cp extract-xml-pages.sh \ + assemble-quickbook.sh \ + prepare-stage1.sh trace-stage1.sh render-stage1.sh \ + execute-stage2.sh \ +build && \ + +cd build && \ + +echo "Calling extract-xml-pages.sh..." && \ +./extract-xml-pages.sh && \ + +echo "Running prepare-stage1.sh..." && \ +./prepare-stage1.sh && \ + +echo "Running trace-stage1.sh..." && \ +./trace-stage1.sh && \ + +echo "Running render-stage1.sh..." && \ +./render-stage1.sh && \ + +echo "Running execute-stage2.sh..." && \ +./execute-stage2.sh && \ + +echo "Calling assemble-quickbook.sh..." && \ +./assemble-quickbook.sh && \ + +echo "Calling the example build to run the Quickbook -> BoostBook -> DocBook -> HTML conversion..." && \ +cd .. && \ +../../../b2.exe + +date diff --git a/example_visualized/stage2-params.xml b/example_visualized/stage2-params.xml new file mode 100644 index 0000000..997c207 --- /dev/null +++ b/example_visualized/stage2-params.xml @@ -0,0 +1,3 @@ + + yes + diff --git a/example_visualized/trace-stage1.sh b/example_visualized/trace-stage1.sh new file mode 100755 index 0000000..339e8d7 --- /dev/null +++ b/example_visualized/trace-stage1.sh @@ -0,0 +1,6 @@ +echo "Removing previous stage1 results..." +rm -rf stage1_visualized/results +mkdir -p stage1_visualized/results + +echo "Tracing stage1..." +java -Xmx1024M -cp "$CLASSPATH" net.sf.saxon.Transform -threads:32 -s:xml-pages -o:stage1_visualized/results -xsl:xslt-visualizer/xsl/run-trace.xsl trace-enabled-stylesheet-uri=../../stage1_visualized/code-trace-enabled/stage1.xsl diff --git a/example_visualized/trace-stage2.sh b/example_visualized/trace-stage2.sh new file mode 100755 index 0000000..917e2da --- /dev/null +++ b/example_visualized/trace-stage2.sh @@ -0,0 +1,11 @@ +#rm -rf stage2_visualized/results + +echo "Removing previous stage2 result traces..." +rm -rf stage2_visualized/results/trace-data +mkdir -p stage2_visualized/results + +echo "Tracing stage2..." +java -Xmx1024M -cp "$CLASSPATH" net.sf.saxon.Transform -threads:32 -s:stage1_visualized/results -o:stage2_visualized/results -xsl:xslt-visualizer/xsl/run-trace.xsl trace-enabled-stylesheet-uri=../../stage2_visualized/code-trace-enabled/stage2.xsl '?transform-params=doc("stage2-params.xml")' principal-output-method=text + +echo "Copying the stage2 results to .txt files (for easier viewing in the browser)..." +java -cp "$CLASSPATH" net.sf.saxon.Transform -s:xml-pages.xml -xsl:debug-friendly-quickbook.xsl input-dir=stage2_visualized/results >/dev/null diff --git a/example_visualized/trace.sh b/example_visualized/trace.sh new file mode 100755 index 0000000..94db368 --- /dev/null +++ b/example_visualized/trace.sh @@ -0,0 +1,2 @@ +./trace-stage1.sh +./trace-stage2.sh diff --git a/xslt-visualizer b/xslt-visualizer new file mode 160000 index 0000000..7c6af53 --- /dev/null +++ b/xslt-visualizer @@ -0,0 +1 @@ +Subproject commit 7c6af536c4610e6b4ffb351662961e13e576834d From 4e4effc0a8818f4ce928bc8d8eba54da87bd990e Mon Sep 17 00:00:00 2001 From: Dmitry Arkhipov Date: Wed, 18 Aug 2021 09:12:46 +0300 Subject: [PATCH 2/2] temp --- docca.jam | 204 +++++++++++++++++++++++++++------ example_visualized/run-all2.sh | 137 ++++++++++++++++++++++ 2 files changed, 308 insertions(+), 33 deletions(-) create mode 100755 example_visualized/run-all2.sh diff --git a/docca.jam b/docca.jam index 94028d8..b56637b 100644 --- a/docca.jam +++ b/docca.jam @@ -10,9 +10,14 @@ import "class" : new ; +import feature ; import doxygen ; +import modules ; +import option ; +import os ; import param ; import path ; +import print ; import project ; import property-set ; import quickbook ; @@ -24,7 +29,13 @@ import type ; .here = [ path.make [ modules.binding $(__name__) ] ] ; .here = $(.here:D) ; +.visualize = [ option.get docca-visualize : : "on" ] ; +.target-suffix = dir ; +if "on" = $(.visualize) +{ + .target-suffix = "$(.target-suffix)-vis" ; +} rule reference ( target : sources * : requirements * : default-build * : usage-requirements * ) @@ -37,7 +48,7 @@ rule reference ( target : sources * : requirements * : default-build * local project = [ project.current ] ; local target-dir = $(target:S=) ; - target-dir = $(target-dir:B=_$(target-dir:B)-dir) ; + target-dir = $(target-dir:B=_$(target-dir:B)-$(.target-suffix)) ; # Generate doxygen configuration file from sources sources = [ sequence.transform fix-source $(project) : $(sources) ] ; @@ -63,55 +74,112 @@ rule reference ( target : sources * : requirements * : default-build * ; $(project).mark-target-as-explicit $(target-dir)/stamp ; - # Adopt as a target index.xml which was created as a side-effect make-explicit $(target-dir)/index.xml $(project) : $(target-dir)/stamp : @docca.null-action ; + #-------------------------------------------------------------------------- + # + # Copy visualization assets + local vis-dir = $(.here)/xslt-visualizer ; + local vis-assets ; + for file in [ path.glob-tree $(.here)/xslt-visualizer/assets : *.* ] + { + local dir = [ path.relative $(file) $(vis-dir) ] ; + dir = $(dir:D) ; + vis-assets += + [ copy $(file) $(project) $(target-dir)/stage1_visualized/code-trace-enabled/$(dir) ] + [ copy $(file) $(project) $(target-dir)/stage2_visualized/code-trace-enabled/$(dir) ] + ; + } + alias $(target-dir)/vis-assets : $(vis-assets) ; + $(project).mark-target-as-explicit $(target-dir)/vis-assets ; + #-------------------------------------------------------------------------- # # Copy the project-specific config XSLT - copy-xsl $(overrides) $(project) $(target-dir)/custom-overrides.xsl ; + copy $(overrides) $(project) $(target-dir) : : custom-overrides.xsl ; # Copy all the XSLT modules to the target directory. # Also, specify their dependencies. local src-dir = $(.here)/include/docca ; - copy-xsl $(src-dir)/common.xsl $(project) $(target-dir) ; - copy-xsl $(src-dir)/base-config.xsl $(project) $(target-dir) ; - copy-xsl $(src-dir)/assemble-quickbook.xsl $(project) $(target-dir) ; + copy $(src-dir)/common.xsl $(project) $(target-dir) ; + copy $(src-dir)/base-config.xsl $(project) $(target-dir) ; + copy $(src-dir)/assemble-quickbook.xsl $(project) $(target-dir) ; + copy $(src-dir)/debug-friendly-quickbook.xsl $(project) $(target-dir) ; - copy-xsl $(src-dir)/base-extract-xml-pages.xsl $(project) $(target-dir) + copy $(src-dir)/base-extract-xml-pages.xsl $(project) $(target-dir) : common.xsl ; - copy-xsl $(src-dir)/base-stage1.xsl $(project) $(target-dir) + copy $(src-dir)/base-stage1.xsl $(project) $(target-dir) : common.xsl ; - copy-xsl $(src-dir)/extract-xml-pages.xsl $(project) $(target-dir) + copy $(src-dir)/extract-xml-pages.xsl $(project) $(target-dir) : base-extract-xml-pages.xsl base-config.xsl custom-overrides.xsl ; - copy-xsl $(src-dir)/stage1.xsl $(project) $(target-dir) + copy $(src-dir)/stage1.xsl $(project) $(target-dir) : base-stage1.xsl base-config.xsl custom-overrides.xsl ; - copy-xsl $(src-dir)/base-stage2.xsl $(project) $(target-dir) + copy $(src-dir)/base-stage2.xsl $(project) $(target-dir) : common.xsl ; - copy-xsl $(src-dir)/stage2.xsl $(project) $(target-dir) + copy $(src-dir)/stage2.xsl $(project) $(target-dir) : base-stage2.xsl base-config.xsl custom-overrides.xsl ; + # Copy visualization XSLT modules to the target directory. + copy $(vis-dir)/xsl/lib/xml-to-string.xsl $(project) $(target-dir)/xsl/lib ; + copy $(vis-dir)/xsl/guid.xsl $(project) $(target-dir)/xsl ; + copy $(vis-dir)/xsl/generate-built-in-rules.xsl $(project) $(target-dir)/xsl ; + copy $(vis-dir)/xsl/do-trace-enable.xsl $(project) $(target-dir)/xsl ; + copy $(vis-dir)/xsl/to-string.xsl $(project) $(target-dir)/xsl ; + copy $(vis-dir)/xsl/html-rule-tree.xsl $(project) $(target-dir)/xsl ; + copy $(vis-dir)/xsl/run-trace.xsl $(project) $(target-dir)/xsl ; + copy $(vis-dir)/xsl/trace-enable.xsl $(project) $(target-dir)/xsl + : lib/xml-to-string.xsl + guid.xsl + generate-built-in-rules.xsl + do-trace-enable.xsl + to-string.xsl + html-rule-tree.xsl + ; + + if $(.visualize) + { + make-explicit $(target-dir)/stage1_visualized/code-trace-enabled/stage1.xsl + $(project) + : $(target-dir)/stage1.xsl + $(target-dir)/xsl/trace-enable.xsl + : @saxonhe.saxonhe + ; + + make-explicit $(target-dir)/stage2_visualized/code-trace-enabled/stage2.xsl + $(project) + : $(target-dir)/stage2.xsl + $(target-dir)/xsl/trace-enable.xsl + : @saxonhe.saxonhe + ; + } + + # Generate stage2 parameters file + make-explicit $(target-dir)/stage2-params.xml $(project) + : + : @docca.generate-stage2-params + ; + #------------------------------------------------------------------------------- # # Run index.xml through the first transformation stage @@ -129,19 +197,51 @@ rule reference ( target : sources * : requirements * : default-build * : @docca.null-action ; - make-explicit $(target-dir)/stage1/results $(project) - : $(target-dir)/xml-pages - $(target-dir)/stage1.xsl - : @saxonhe.saxonhe_dir - : $(requirements) - ; - - make-explicit $(target-dir)/stage2/results $(project) - : $(target-dir)/stage1/results - $(target-dir)/stage2.xsl - : @saxonhe.saxonhe_dir - : $(requirements) - ; + if $(.visualize) + { + make-explicit $(target-dir)/stage1/results $(project) + : $(target-dir)/xml-pages + $(target-dir)/xsl/run-trace.xsl + : @docca.saxonhe-dir + : $(requirements) + trace-enabled-stylesheet-uri=../stage1_visualized/code-trace-enabled/stage1.xsl + ; + + make-explicit $(target-dir)/stage2/results $(project) + : $(target-dir)/stage1/results + $(target-dir)/xsl/run-trace.xsl + : @docca.saxonhe-dir + : $(requirements) + trace-enabled-stylesheet-uri=../stage2_visualized/code-trace-enabled/stage2.xsl + "?transform-params=doc(\"stage2-params.xml\")" + principal-output-method=text + ; + + make-explicit $(target-dir)/stage2/results/copied.stamp $(project) + : $(target-dir)/xml-pages.xml + $(target-dir)/debug-friendly-quickbook.xsl + : @docca.saxonhe + : $(requirements) + $(target-dir)/stage2/results + input-dir=stage2_visualized/results + ; + } + else + { + make-explicit $(target-dir)/stage1/results $(project) + : $(target-dir)/xml-pages + $(target-dir)/stage1-prepared.xsl + : @docca.saxonhe-dir + : $(requirements) + ; + + make-explicit $(target-dir)/stage2/results $(project) + : $(target-dir)/stage1/results + $(target-dir)/stage2.xsl + : @docca.saxonhe-dir + : $(requirements) + ; + } generate $(target) : $(target-dir)/xml-pages.xml @@ -179,25 +279,22 @@ rule make-qbk ( project name : property-set : sources * ) } -local rule copy-xsl ( source project target-or-dir : dependencies * ) +local rule copy ( source project target-dir : dependencies * : new-name ? ) { local target ; - local dir ; - if .xsl = $(target-or-dir:S) + if $(new-name) { - dir = $(target-or-dir:D) ; - target = $(target-or-dir:D=) ; + target = $(new-name) ; } else { - dir = $(target-or-dir) ; target = $(source:D=) ; } - make-explicit $(target:TD=$(dir)) $(project) + make-explicit $(target:TD=$(target-dir)) $(project) : $(source) : @common.copy - : $(dependencies:TD=$(dir)) + : $(target-dir)/$(dependencies) ; } @@ -207,6 +304,8 @@ local rule make-explicit ( target project : sources * : make-rule + : reqs * { make $(target) : $(sources) : $(make-rule) : $(reqs) : $(ureqs) ; $(project).mark-target-as-explicit $(target) ; + + return $(target) ; } @@ -229,3 +328,42 @@ local rule fix-source ( project path ) rule null-action { } + + +rule generate-stage2-params ( target : sources * : properties * ) +{ + print.output $(target) ; + print.text + "" + " yes" + "" + : overwrite + ; +} + + +.java_exe = [ modules.peek saxonhe : .java_exe ] ; +.saxonhe_jar = [ modules.peek saxonhe : .saxonhe_jar ] ; + +toolset.flags docca.saxonhe-dir FLAGS : ; + +actions saxonhe +{ + "$(.java_exe)" -jar "$(.saxonhe_jar)" -o:"$(<)" -s:"$(>[1])" -xsl:"$(>[2])" $(FLAGS) +} +if [ os.on-windows ] +{ + actions saxonhe-dir + { + if not exist "$(<)\\" mkdir "$(<)" + "$(.java_exe)" -jar "$(.saxonhe_jar)" -o:"$(<)" -s:"$(>[1])" -xsl:"$(>[2])" $(FLAGS) + } +} +else +{ + actions saxonhe-dir + { + mkdir -p "$(<)" + "$(.java_exe)" -jar "$(.saxonhe_jar)" -o:"$(<)" -s:"$(>[1])" -xsl:"$(>[2])" $(FLAGS) + } +} diff --git a/example_visualized/run-all2.sh b/example_visualized/run-all2.sh new file mode 100755 index 0000000..1ff4081 --- /dev/null +++ b/example_visualized/run-all2.sh @@ -0,0 +1,137 @@ +!/bin/bash + +: ${VISUALIZE:=0} + +echo "Copying docca XSLT files..." && \ +cp ../include/docca/* build + +echo "Copying shell scripts..." && \ +cp extract-xml-pages.sh \ + prepare.sh prepare-stage1.sh prepare-stage2.sh \ + trace.sh trace-stage1.sh trace-stage2.sh \ + render.sh render-stage1.sh render-stage2.sh \ + assemble-quickbook.sh \ +build + +if [ VISUALIZE != 0 ]; then + echo "Copying xslt-visualizer..." && \ + rm -rf build/xslt-visualizer && \ + cp -r ../xslt-visualizer build +fi + +if [ VISUALIZE = 2 ]; then + echo "Copying stage2 parameters file..." && \ + cp stage2-params.xml build +fi + +cd build && \ + +echo "Calling extract-xml-pages.sh..." && \ +./extract-xml-pages.sh + +echo "Removing previous stage1 results..." +rm -rf stage1_visualized/results +mkdir -p stage1_visualized/results + +if [ VISUALIZE = 0 ]; then + STAGE1_EXEC_PARAMS = -xsl:stage1.xsl +else + # prepare-stage1.sh + rm -rf stage1_visualized/code-trace-enabled + java -cp "$CLASSPATH" net.sf.saxon.Transform -s:stage1.xsl -o:stage1_visualized/code-trace-enabled/stage1.xsl -xsl:xslt-visualizer/xsl/trace-enable.xsl + + STAGE1_EXEC_PARAMS = -xsl:xslt-visualizer/xsl/run-trace.xsl trace-enabled-stylesheet-uri=../../stage1_visualized/code-trace-enabled/stage1.xsl +fi + +echo "Executing stage1..." +# echo "Tracing stage1..." +java -Xmx1024M -cp "$CLASSPATH" net.sf.saxon.Transform -threads:32 -s:xml-pages -o:stage1_visualized/results $STAGE1_EXEC_PARAMS + +if [ VISUALIZE != 0 ]; then + # render-stage1.sh + mkdir -p stage1_visualized/visualized/assets + cp -ru xslt-visualizer/assets/* stage1_visualized/visualized/assets + + echo "Rendering the stage1 visualization results" + # Create the rendered results (Saxon forces .xml prefix in the output files) + java -cp "$CLASSPATH" net.sf.saxon.Transform -threads:128 -s:stage1_visualized/results/trace-data -o:stage1_visualized/visualized -xsl:xslt-visualizer/xsl/render.xsl + + echo "Renaming the stage1 visualization files to .html" + cd stage1_visualized/visualized + for file in *.xml + do + mv "$file" "${file%.xml}.html" + done + cd ../.. +fi + +if [ VISUALIZE = 2 ]; then + # prepare-stage2.sh + rm -rf stage2_visualized/code-trace-enabled + java -cp "$CLASSPATH" net.sf.saxon.Transform -s:stage2.xsl -o:stage2_visualized/code-trace-enabled/stage2.xsl -xsl:xslt-visualizer/xsl/trace-enable.xsl + + # trace-stage2.sh + echo "Removing previous stage2 result traces..." + rm -rf stage2_visualized/results/trace-data + mkdir -p stage2_visualized/results + + echo "Tracing stage2..." + java -Xmx1024M -cp "$CLASSPATH" net.sf.saxon.Transform -threads:32 -s:stage1_visualized/results -o:stage2_visualized/results -xsl:xslt-visualizer/xsl/run-trace.xsl trace-enabled-stylesheet-uri=../../stage2_visualized/code-trace-enabled/stage2.xsl '?transform-params=doc("stage2-params.xml")' principal-output-method=text + + echo "Copying the stage2 results to .txt files (for easier viewing in the browser)..." + java -cp "$CLASSPATH" net.sf.saxon.Transform -s:xml-pages.xml -xsl:debug-friendly-quickbook.xsl input-dir=stage2_visualized/results >/dev/null + + # render-stage2.sh + mkdir -p stage2_visualized/visualized/assets + cp -ru xslt-visualizer/assets/* stage2_visualized/visualized/assets + + echo "Rendering the stage2 visualization results" + java -cp "$CLASSPATH" net.sf.saxon.Transform -threads:128 -s:stage2_visualized/results/trace-data -o:stage2_visualized/visualized -xsl:xslt-visualizer/xsl/render.xsl + + echo "Renaming the stage2 visualization files to .html" + cd stage2_visualized/visualized + for file in *.xml + do + mv "$file" "${file%.xml}.html" + done +fi + +if [ VISUALIZE = 0 ]; then + # execute-stage2.sh + mkdir -p stage2 + + echo "Executing stage2..." + java -cp "$CLASSPATH" net.sf.saxon.Transform -threads:128 -s:stage1_visualized/results -o:stage2_visualized/results -xsl:stage2.xsl DEBUG=yes + + echo "Copying the stage2 results to .txt files (for easier viewing in the browser)..." + java -cp "$CLASSPATH" net.sf.saxon.Transform -s:xml-pages.xml -xsl:debug-friendly-quickbook.xsl input-dir=stage2_visualized/results >/dev/null +fi + +echo "Copying a manually-modified xml-pages.xml if present..." +cp ../xml-pages.xml . + +echo "Assembling the stage2 results into reference.qbk" +java -cp "$CLASSPATH" net.sf.saxon.Transform -s:xml-pages.xml -xsl:assemble-quickbook.xsl -o:reference.qbk input-dir=stage2_visualized/results + +echo "Copying reference.qbk..." +cp reference.qbk .. + +echo "Calling the example build to run the Quickbook -> BoostBook -> DocBook -> HTML conversion..." && \ +cd .. && \ +../../../b2.exe + +# Everything below tries to replicate what the b2.exe call does above +#echo "Copying reference.qbk into qbk..." && \ +#cp ../reference.qbk ../qbk && \ + +#echo "Converting QuickBook (main.qbk) to BoostBook (beast_doc.xml)..." && \ +#../../../../bin.v2/tools/quickbook/src/msvc-14.2/release/cxxstd-0x-iso/link-static/threading-multi/quickbook.exe --output-file=beast_doc.xml ../qbk/main.qbk && \ + +#echo "Converting BoostBook (beast_doc.xml) to DocBook (beast_doc.docbook)..." && \ +#set XML_CATALOG_FILES=../../../../bin.v2/boostbook_catalog.xml && \ +#/usr/bin/xsltproc --stringparam boost.defaults "Boost" --stringparam boost.root "../../../.." --stringparam chapter.autolabel "1" --stringparam chunk.first.sections "1" --stringparam chunk.section.depth "8" --stringparam generate.section.toc.level "8" --stringparam generate.toc "chapter toc,title section nop reference nop" --stringparam toc.max.depth "8" --stringparam toc.section.depth "8" --path "../../../../bin.v2" --xinclude -o beast_doc.docbook ../../../../tools/boostbook/xsl/docbook.xsl beast_doc.xml && \ + + +#echo "Converting DocBook (beast_doc.docbook) to HTML..." && \ +#set XML_CATALOG_FILES=../../../../bin.v2/boostbook_catalog.xml && \ +#/usr/bin/xsltproc --stringparam boost.defaults "Boost" --stringparam boost.root "../../../.." --stringparam chapter.autolabel "1" --stringparam chunk.first.sections "1" --stringparam chunk.section.depth "8" --stringparam generate.section.toc.level "8" --stringparam generate.toc "chapter toc,title section nop reference nop" --stringparam manifest "beast_HTML.manifest" --stringparam toc.max.depth "8" --stringparam toc.section.depth "8" --path "../../../../bin.v2" --xinclude -o ../html/ ../../../../tools/boostbook/xsl/html.xsl beast_doc.docbook