From efb1d9c1e94cac5c12f7084eb942cbecff7a348b Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 1 May 2026 14:13:56 -0500 Subject: [PATCH 1/4] add personography class --- lib/datura/data_manager.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 24f4898e9..336a17bf6 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -22,6 +22,7 @@ def self.format_to_class "csv" => FileCsv, "ead" => FileEad, "html" => FileHtml, + "personography" => FilePersonography, "tei" => FileTei, "vra" => FileVra, "webs" => FileWebs, From a3f3102cc37b4538bf3934926ef0de7fea3f9e0b Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 1 May 2026 15:03:54 -0500 Subject: [PATCH 2/4] extend personography subclass to handle one to multiple transformation --- .../tei_to_es/tei_to_es_personography.rb | 89 +++++++++++++++---- 1 file changed, 72 insertions(+), 17 deletions(-) diff --git a/lib/datura/to_es/tei_to_es/tei_to_es_personography.rb b/lib/datura/to_es/tei_to_es/tei_to_es_personography.rb index 4d3d43de2..0323fd588 100644 --- a/lib/datura/to_es/tei_to_es/tei_to_es_personography.rb +++ b/lib/datura/to_es/tei_to_es/tei_to_es_personography.rb @@ -1,33 +1,57 @@ +# TeiToEsPersonography transforms a single element from a TEI +# personography file into an Elasticsearch JSON document. +# +# Each element becomes one ES document, identified by the person's +# xml:id attribute value (e.g., "hickok.j"). The parent TEI document is passed +# as @parent_xml so that header-level fields (creator, contributor, etc.) can +# still be extracted from the full file. + class TeiToEsPersonography < TeiToEs - def override_xpaths - { + # Merges person-level xpaths on top of the standard TeiToEs xpaths, then + # merges override_xpaths on top so collection teams can still customize using + # the familiar hook pattern (e.g., in scripts/overrides/tei_to_es_personography.rb). + # Paths here are relative to the element (@xml), not the document root. + # Header-level fields (creator, contributor, rights, etc.) are unchanged and + # are queried against @parent_xml in the field methods below. + def xpaths_list + super.merge({ "titles" => { "main" => "persName[@type='display']", - "alt" => "persName" + "alt" => "persName" }, - "text" => "note" - } + "text" => "note", + "birth_year" => "birth/@when", + "death_year" => "death/@when" + }).merge(override_xpaths) + end + + # No-op by default; override in a collection script to add or replace xpaths. + def override_xpaths + {} + end + + # Returns the person's xml:id value as the document identifier. + # After namespace removal by Nokogiri, xml:id is accessible as the "id" attribute. + def get_id + @xml["id"] end def category "Life" end - def creator - creators = get_list(@xpaths["creators"], false, @parent_xml) - if creators - creators.map { |c| { "name" => c } } - end + # Hardcoded subcategory for all personography entries. + def subcategory + "Personography" end - def creators - get_text(@xpaths["creators"], false, @parent_xml) + def creator + [] end - def get_id - person = @xml["id"] - "#{@filename}_#{person}" + def creators + nil end def person @@ -38,8 +62,39 @@ def person }] end - def subcategory - "Personography" + # Uses birth/@when as the person's primary date (standardized to start of year). + def date + year = get_text(@xpaths["birth_year"]) + Datura::Helpers.date_standardize(year, true) if year + end + + # Same as date — birth year as the not-before bound. + def date_not_before + year = get_text(@xpaths["birth_year"]) + Datura::Helpers.date_standardize(year, true) if year + end + + # Uses death/@when as the not-after date (standardized to end of year). + def date_not_after + year = get_text(@xpaths["death_year"]) + Datura::Helpers.date_standardize(year, false) if year + end + + # Raw birth year string for display (e.g., "1850"). + def date_display + get_text(@xpaths["birth_year"]) + end + + # Points to the source personography XML file (the whole file, not per-person), + # since all persons share one source document. + def uri_data + File.join( + @options["data_base"], + "data", + @options["collection"], + "source/personography", + "#{@filename}.xml" + ) end end From ad32b1ef1a10c56b17a48f5aa08cd7cc56b0d892 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 1 May 2026 15:15:42 -0500 Subject: [PATCH 3/4] build new personography file type and update filename var in xslt --- lib/datura/file_types/file_personography.rb | 100 ++++++++++++++++++ .../lib/personography_encyclopedia.xsl | 4 +- 2 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 lib/datura/file_types/file_personography.rb diff --git a/lib/datura/file_types/file_personography.rb b/lib/datura/file_types/file_personography.rb new file mode 100644 index 000000000..5017e026c --- /dev/null +++ b/lib/datura/file_types/file_personography.rb @@ -0,0 +1,100 @@ +require_relative "../helpers.rb" +require_relative "../file_type.rb" +require_relative "../solr_poster.rb" +require_relative "file_tei.rb" +require "rest-client" +require "json" + +# FilePersonography handles TEI personography files — XML files containing +# a element with individual children, each identified +# by a unique xml:id attribute. +# +# Unlike standard TEI processing (one ES document per file), this class +# produces one ES JSON document per element, with each output file +# named by the person's xml:id value (e.g., hickok.j.json). +# +# HTML output uses the standard tei_to_html XSLT, which via +# personography_encyclopedia.xsl writes: +# - one combined HTML file with all persons as divs (served at /item/{filename}) +# - individual HTML files per person via xsl:result-document (served at /item/{xml:id}) +# +# Solr output uses the standard tei_to_solr XSLT, which via +# tei_to_solr/lib/personography.xsl produces per-person Solr docs. +# +# Source files should be placed in source/personography/ within the collection. +class FilePersonography < FileTei + + # Maps each element to TeiToEsPersonography for ES transformation. + # The parent /TEI document is intentionally excluded — one ES doc is produced + # per person, not one for the entire file. + def subdoc_xpaths + { "//listPerson/person" => TeiToEsPersonography } + end + + # Overrides FileType#transform_es to write one JSON file per person + # (named by the person's xml:id) instead of a single file for the whole document. + # Returns an array of JSON hashes so that post_es can iterate and POST them + # to Elasticsearch without modification. + def transform_es + es_req = [] + begin + file_xml = parse_markup_lang_file + + # Verify at least one person element exists before continuing + results = file_xml.xpath(*subdoc_xpaths.keys) + if results.length == 0 + raise "No persons found in #{self.filename}; verify //listPerson/person matches the XML structure" + end + + file_xml.xpath("//listPerson/person").each do |person_node| + transformer = TeiToEsPersonography.new(person_node, @options, file_xml, self.filename(false)) + person_json = transformer.json + es_req << person_json + + # Write one JSON file per person using the person's identifier as the filename. + # Produces e.g. output/es/hickok.j.json rather than output/es/wfc.person.json. + if @options["output"] + person_id = person_json["identifier"] + if person_id.nil? || person_id.empty? + puts "WARNING: person in #{self.filename} has no identifier; skipping ES file output" + next + end + filepath = File.join(@out_es, "#{person_id}.json") + File.open(filepath, "w") { |f| f.write(JSON.pretty_generate(person_json)) } + end + end + + return es_req + rescue => e + puts "Error transforming #{self.filename}: #{e}" + puts e.backtrace + raise e + end + end + + # Overrides the default transform_html to use Saxon's -o: flag instead of + # piping to tee. This is necessary because xsl:result-document hrefs in the + # personography XSLT are relative paths — Saxon resolves them relative to the + # -o: output file's directory, which ensures individual per-person HTML files + # land in @out_html alongside the combined file. + # + # To suppress the combined single-file HTML output entirely, remove the -o: + # flag from the command (or add this override to a project override file and + # omit that line). + def transform_html + params = CommonXml.stringify_params(@options["variables_html"]) + outfile = File.join(@out_html, "#{self.filename(false)}.html") + cmd = "saxon -s:#{@file_location} -xsl:#{@script_html} -o:#{outfile} #{params}" + puts "using command #{cmd}" if @options["verbose"] + Open3.popen3(cmd) do |stdin, stdout, stderr| + err = stderr.read + return { "error" => err } if err && !err.empty? + { "doc" => stdout.read } + end + end + + def transform_iiif + raise "Personography to IIIF is not implemented" + end + +end diff --git a/lib/xslt/tei_to_html/lib/personography_encyclopedia.xsl b/lib/xslt/tei_to_html/lib/personography_encyclopedia.xsl index d15066563..04209d066 100644 --- a/lib/xslt/tei_to_html/lib/personography_encyclopedia.xsl +++ b/lib/xslt/tei_to_html/lib/personography_encyclopedia.xsl @@ -39,8 +39,8 @@ - - + + From 70b5c76484c9cfb44f8bb75b6bc6fc5438a0f4ab Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 1 May 2026 17:05:56 -0500 Subject: [PATCH 4/4] adjust filename var in xslt, remove now unnecessary override --- lib/datura/file_types/file_personography.rb | 21 ------------------- .../lib/personography_encyclopedia.xsl | 4 ++-- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/lib/datura/file_types/file_personography.rb b/lib/datura/file_types/file_personography.rb index 5017e026c..b56ba2f9f 100644 --- a/lib/datura/file_types/file_personography.rb +++ b/lib/datura/file_types/file_personography.rb @@ -72,27 +72,6 @@ def transform_es end end - # Overrides the default transform_html to use Saxon's -o: flag instead of - # piping to tee. This is necessary because xsl:result-document hrefs in the - # personography XSLT are relative paths — Saxon resolves them relative to the - # -o: output file's directory, which ensures individual per-person HTML files - # land in @out_html alongside the combined file. - # - # To suppress the combined single-file HTML output entirely, remove the -o: - # flag from the command (or add this override to a project override file and - # omit that line). - def transform_html - params = CommonXml.stringify_params(@options["variables_html"]) - outfile = File.join(@out_html, "#{self.filename(false)}.html") - cmd = "saxon -s:#{@file_location} -xsl:#{@script_html} -o:#{outfile} #{params}" - puts "using command #{cmd}" if @options["verbose"] - Open3.popen3(cmd) do |stdin, stdout, stderr| - err = stderr.read - return { "error" => err } if err && !err.empty? - { "doc" => stdout.read } - end - end - def transform_iiif raise "Personography to IIIF is not implemented" end diff --git a/lib/xslt/tei_to_html/lib/personography_encyclopedia.xsl b/lib/xslt/tei_to_html/lib/personography_encyclopedia.xsl index 04209d066..337a7bd01 100644 --- a/lib/xslt/tei_to_html/lib/personography_encyclopedia.xsl +++ b/lib/xslt/tei_to_html/lib/personography_encyclopedia.xsl @@ -39,8 +39,8 @@ - - + +