Closes #632

teamengine-core/src/main/java/com/occamlab/te/Generator.java

@@ -14,6 +14,8 @@ C. Heazel (WiSC): Added Fortify adjudication changes
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.InputStream;
+import java.net.URL;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.logging.Level;
@@ -103,38 +105,63 @@ private static Index generateXsl(SetupOptions opts, String generatorStylesheetRe
 		XsltTransformer generatorTransformer = generatorXsltExecutable.load();
 
 		// Create a list of CTL sources (may be files or dirs)
-		ArrayList<File> sources = new ArrayList<>();
+		ArrayList<Object> sources = new ArrayList<>();
 		File f = Misc.getResourceAsFile("com/occamlab/te/scripts/parsers.ctl");
 		if (f.exists()) {
 			sources.add(f.getParentFile());
 		}
+		else {
+			sources.add(new URL(Misc.getResourceURL("com/occamlab/te/scripts/parsers.ctl")));
+			sources.add(new URL(Misc.getResourceURL("com/occamlab/te/scripts/functions.ctl")));
+		}
 		sources.addAll(opts.getSources());
 
 		// Create a list of source CTL files only (no dirs),
 		// and a corresponding list containing a working dir for each file
-		ArrayList<File> sourceFiles = new ArrayList<>();
+		ArrayList<Object> sourceObjects = new ArrayList<>();
 		ArrayList<File> workDirs = new ArrayList<>();
-		Iterator<File> it = sources.iterator();
+		Iterator<Object> it = sources.iterator();
 		while (it.hasNext()) {
-			File source = it.next();
-			LOGR.log(Level.FINE, "Processing CTL source files in {0}", source.getAbsolutePath());
-			String encodedName = createEncodedName(source);
+			Object source = it.next();
+			File sourceFile = null;
+			URL sourceURL = null;
+			if (source instanceof File) {
+				sourceFile = (File) source;
+			}
+			else if (source instanceof URL) {
+				sourceURL = (URL) source;
+			}
+
+			String encodedName = "";
+			if (sourceFile != null) {
+				LOGR.log(Level.FINE, "Processing CTL source files in {0}", sourceFile.getAbsolutePath());
+				encodedName = createEncodedName(sourceFile.toURI().toString());
+			}
+			if (sourceURL != null) {
+				LOGR.log(Level.FINE, "Processing CTL source url {0}", sourceURL.toString());
+				encodedName = createEncodedName(sourceURL.toString());
+			}
 			if (docMode) {
 				encodedName += "d";
 			}
 			File workingDir = new File(opts.getWorkDir(), encodedName);
 			if (!workingDir.exists() && !workingDir.mkdir()) {
 				LOGR.log(Level.WARNING, "Unable to create working directory at {0}", workingDir.getAbsolutePath());
 			}
-			if (source.isDirectory()) {
-				String[] children = source.list();
+			if (sourceURL != null) {
+				sourceObjects.add(sourceURL);
+				workDirs.add(workingDir);
+				continue;
+			}
+			if (sourceFile.isDirectory()) {
+				String[] children = sourceFile.list();
 				for (int i = 0; i < children.length; i++) {
 					// Finds all .ctl and .xml files in the directory to use
 					String lowerName = children[i].toLowerCase();
 					if (lowerName.endsWith(".ctl") || lowerName.endsWith(".xml")) {
-						File file = new File(source, children[i]);
+						File file = new File(sourceFile, children[i]);
 						if (file.isFile()) {
-							sourceFiles.add(file);
+							sourceObjects.add(file);
 							String basename = children[i].substring(0, children[i].length() - 4);
 							File subdir = new File(workingDir, basename);
 							subdir.mkdir();
@@ -144,7 +171,7 @@ private static Index generateXsl(SetupOptions opts, String generatorStylesheetRe
 				}
 			}
 			else {
-				sourceFiles.add(source);
+				sourceObjects.add(sourceFile);
 				workDirs.add(workingDir);
 			}
 		}
@@ -155,8 +182,16 @@ private static Index generateXsl(SetupOptions opts, String generatorStylesheetRe
 		File generatorStylesheet = Misc.getResourceAsFile(generatorStylesheetResource);
 
 		// Process each CTL source file
-		for (int i = 0; i < sourceFiles.size(); i++) {
-			File sourceFile = sourceFiles.get(i);
+		for (int i = 0; i < sourceObjects.size(); i++) {
+			Object sourceObject = sourceObjects.get(i);
+			File sourceFile = null;
+			URL sourceURL = null;
+			if (sourceObject instanceof File) {
+				sourceFile = (File) sourceObject;
+			}
+			else if (sourceObject instanceof URL) {
+				sourceURL = (URL) sourceObject;
+			}
 			File workingDir = workDirs.get(i);
 
 			// Read previous index for this file (if any), and determine whether
@@ -189,17 +224,30 @@ else if (indexFile.isFile()) {
 				boolean validationErrors = false;
 				if (opts.isValidate()) {
 					int old_count = validation_eh.getErrorCount();
-					LOGR.log(Level.CONFIG, "Validating " + sourceFile);
-					ctl_validator.validate(new StreamSource(sourceFile));
+					if (sourceFile != null) {
+						LOGR.log(Level.CONFIG, "Validating " + sourceFile);
+						ctl_validator.validate(new StreamSource(sourceFile));
+					}
+					if (sourceURL != null) {
+						LOGR.log(Level.CONFIG, "Validating " + sourceURL);
+						ctl_validator.validate(new StreamSource((InputStream) sourceURL.getContent()));
+					}
 					validationErrors = (validation_eh.getErrorCount() > old_count);
 				}
 
 				if (!validationErrors) {
 					// Clean up the working directory
 					Misc.deleteDirContents(workingDir);
 
-					InputSource input = new InputSource(new FileInputStream(sourceFile));
-					input.setSystemId(sourceFile.toURI().toString());
+					InputSource input = null;
+					if (sourceFile != null) {
+						input = new InputSource(new FileInputStream(sourceFile));
+						input.setSystemId(sourceFile.toURI().toString());
+					}
+					if (sourceURL != null) {
+						input = new InputSource((InputStream) sourceURL.getContent());
+						input.setSystemId(sourceURL.toString());
+					}
 					// Fortify Mods to prevent External Entity Injection
 					XMLReader reader = parser.getXMLReader();
 					reader.setFeature("http://xml.org/sax/features/external-general-entities", false);
@@ -244,14 +292,13 @@ else if (indexFile.isFile()) {
 
 	/**
 	 * Creates a directory name from a file path.
-	 * @param source A File reference.
+	 * @param reference A reference (File or URL).
 	 * @return A String representing a legal directory name.
 	 */
-	public static String createEncodedName(File source) {
-		String fileURI = source.toURI().toString();
+	public static String createEncodedName(String reference) {
 		String userDirURI = new File(System.getProperty("user.dir")).toURI().toString();
-		fileURI = fileURI.replace(userDirURI, "");
-		return fileURI.substring(fileURI.lastIndexOf(':') + 1).replace("%20", "-").replace('/', '_');
+		reference = reference.replace(userDirURI, "");
+		return reference.substring(reference.lastIndexOf(':') + 1).replace("%20", "-").replace('/', '_');
 	}
 
 }

teamengine-core/src/main/java/com/occamlab/te/SetupOptions.java

@@ -61,7 +61,7 @@ public class SetupOptions {
 
 	String sourcesName = "default";
 
-	ArrayList<File> sources = new ArrayList<>();
+	ArrayList<Object> sources = new ArrayList<>();
 
 	private static Logger jLogger = Logger.getLogger("com.occamlab.te.SetupOptions");
 
@@ -170,22 +170,21 @@ public File getWorkDir() {
 	}
 
 	/**
-	 * Returns a list of file system resources (directories and files) containing CTL test
-	 * scripts.
-	 * @return A List containing one or more File references (TE_BASE/scripts is the
-	 * default location).
+	 * Returns a list of file references (Files or URLs) containing CTL test scripts.
+	 * @return A List containing one or more references (Files or URLs) (TE_BASE/scripts
+	 * is the default location).
 	 */
-	public List<File> getSources() {
+	public List<Object> getSources() {
 		return sources;
 	}
 
 	/**
 	 * Adds a file system resource to the collection of known scripts.
-	 * @param source A File object representing a file or directory.
+	 * @param source An Object representing a file or directory or a URL.
 	 * @deprecated Use {@link SetupOptions#addSourceWithValidation(File source)} instead
 	 */
 	@Deprecated
-	public void addSource(File source) {
+	public void addSource(Object source) {
 		this.sources.add(source);
 	}
 

teamengine-core/src/main/java/com/occamlab/te/Test.java

@@ -362,8 +362,12 @@ else if (mode == DOC_MODE) {
 			masterIndex = new Index();
 		}
 		List<File> resourcesDirs = new ArrayList<>();
-		for (File sourceFile : setupOpts.getSources()) {
-			File resourcesDir = findResourcesDirectory(sourceFile);
+		for (Object sourceObject : setupOpts.getSources()) {
+			if (!(sourceObject instanceof File)) {
+				LOGR.info("Skipping adding sourceObject to resourceDirs: " + sourceObject);
+				continue;
+			}
+			File resourcesDir = findResourcesDirectory((File) sourceObject);
 			if (!resourcesDirs.contains(resourcesDir)) {
 				resourcesDirs.add(resourcesDir);
 			}
@@ -487,8 +491,12 @@ private void saveSources() {
 			File sourcesFile = new File(sessionDir, "sources.xml");
 			try (PrintWriter writer = new PrintWriter(sourcesFile)) {
 				writer.println("<sources>");
-				for (File file : setupOpts.getSources()) {
-					writer.println("<source>" + file.getPath() + "</source>");
+				for (Object object : setupOpts.getSources()) {
+					if (!(object instanceof File)) {
+						LOGR.info("Skipping adding source to sources.xml: " + object);
+						continue;
+					}
+					writer.println("<source>" + ((File) object).getPath() + "</source>");
 				}
 				writer.println("</sources>");
 			}

teamengine-core/src/main/java/com/occamlab/te/util/DocumentationHelper.java

@@ -195,7 +195,14 @@ else if (args[i].equals("-h") || args[i].equals("-help") || args[i].equals("-?")
 			throw new Exception("Error: Documentation file already exists, check the file "
 					+ html_output_documentation_file.getAbsolutePath() + " ");
 		FileOutputStream fos = new FileOutputStream(html_output_documentation_file);
-		docCode.generateDocumentation(setupOpts.getSources().get(0).getAbsolutePath(), fos);
+		Object source = setupOpts.getSources().get(0);
+		if (source instanceof File) {
+			docCode.generateDocumentation(((File) source).getAbsolutePath(), fos);
+		}
+		else if (source instanceof URL) {
+			docCode.generateDocumentation(((URL) source).toString(), fos);
+		}
+
 		fos.close();
 		System.out
 			.println("Test documentation file \"" + html_output_documentation_file.getAbsolutePath() + "\" created!");

teamengine-core/src/test/java/com/occamlab/te/SetupOptionsTest.java

@@ -65,7 +65,7 @@ public void getWorkDir() {
 	@Test
 	public void getScriptSources() {
 		SetupOptions iut = new SetupOptions();
-		List<File> sources = iut.getSources();
+		List<Object> sources = iut.getSources();
 		Assert.assertEquals("Unexpected size", 0, sources.size());
 	}
 

teamengine-web/src/main/java/com/occamlab/te/web/TestServlet.java

@@ -185,7 +185,7 @@ public void init() throws ServletException {
 				indexes.put(sourcesName, index);
 
 				for (File ctlFile : index.getDependencies()) {
-					String encodedName = Generator.createEncodedName(ctlFile);
+					String encodedName = Generator.createEncodedName(ctlFile.toURI().toString());
 					String basename = encodedName;
 					int i = basename.lastIndexOf('.');
 					if (i > 0) {