diff --git a/docker-compose-official.yml b/docker-compose-official.yml new file mode 100644 index 00000000..2bc91b7e --- /dev/null +++ b/docker-compose-official.yml @@ -0,0 +1,30 @@ +services: + faction-mongo: + image: mongo:8.0 + ports: + - "27017:27017" + volumes: + - "~/.faction/data-dev:/data/db" + tomcat-service: + image: factionsecurityllc/owasp-faction:latest + volumes: + # Mount exploded WAR directory for hot reload + - ./target/faction:/usr/local/tomcat/webapps/ROOT + environment: + - "FACTION_MONGO_HOST=faction-mongo" + - "FACTION_MONGO_DATABASE=faction" + - "FACTION_SECRET_KEY=faction_encryption_key" + - "FACTION_OAUTH_CALLBACK=http://localhost:8080" + # Enable JVM hot reload + remote debugger (JDWP) on port 8000. + # Attach your IDE to localhost:8000; suspend=n so the app starts normally. + - "JAVA_OPTS=-Dfile.encoding=UTF-8 -Xms512m -Xmx1024m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:8000" + links: + - faction-mongo + command: ["/usr/local/tomcat/bin/catalina.sh", "run"] + ports: + - "8080:8080" + - "8000:8000" + # Optional: JMX port for monitoring + - "9090:9090" + depends_on: + - faction-mongo diff --git a/src/com/fuse/actions/assessment/AddVulnerability.java b/src/com/fuse/actions/assessment/AddVulnerability.java index 6da525e9..1619da2a 100644 --- a/src/com/fuse/actions/assessment/AddVulnerability.java +++ b/src/com/fuse/actions/assessment/AddVulnerability.java @@ -43,11 +43,14 @@ import com.fuse.dao.Note; import com.fuse.dao.PeerReview; import com.fuse.dao.ReportMap; +import com.fuse.dao.ReportOptions; import com.fuse.dao.User; import com.fuse.dao.Vulnerability; import com.fuse.dao.query.VulnerabilityQueries; +import com.fuse.reporting.DocxPrecompiler; import com.fuse.servlets.EventStreamServlet; import com.fuse.events.VulnerabilityEventBus; +import com.fuse.utils.FSUtils; import com.fuse.utils.ParseXML; import com.opencsv.*; @@ -385,6 +388,15 @@ public String updateVuln() throws ParseException, UnsupportedEncodingException { HibHelper.getInstance().commit(); this.broadcastVulnUpdates(user, vuln); VulnerabilityEventBus.fire(vuln.getAssessmentId(), this.vulnid); + // pre-compile HTML fields so report generation can skip the + // expensive XHTML conversion for this vuln + { + ReportOptions rpo = FSUtils.getOrCreateReportOptionsIfNotExist(em); + DocxPrecompiler.precompileAndPersist(vuln, + rpo != null ? rpo.getFont() : "Calibri", + rpo != null ? rpo.getBodyCss() : "", + assessment); + } return SUCCESSJSON; } diff --git a/src/com/fuse/actions/assessment/AssessmentView.java b/src/com/fuse/actions/assessment/AssessmentView.java index 0edd047b..9cef3f26 100644 --- a/src/com/fuse/actions/assessment/AssessmentView.java +++ b/src/com/fuse/actions/assessment/AssessmentView.java @@ -910,6 +910,15 @@ private String finalizeAssessment(EntityManager em, Assessment assessment, User List vulns = assessment.getVulns(); for (Vulnerability v : vulns) { v.setOpened(new Date()); + // clear the pre-compiled OOXML cache — finalized assessments + // never regenerate reports, so the cache is dead weight + v.setCachedDescXml(null); + v.setCachedRecXml(null); + v.setCachedDetailsXml(null); + v.setCachedDescHash(null); + v.setCachedRecHash(null); + v.setCachedDetailsHash(null); + v.setCachedCfXml(null); } List notifiers = new ArrayList(); for (User a : assessment.getAssessor()) { diff --git a/src/com/fuse/actions/bootstrap/AppBootstrapListener.java b/src/com/fuse/actions/bootstrap/AppBootstrapListener.java index 2a7337c5..4969c43e 100644 --- a/src/com/fuse/actions/bootstrap/AppBootstrapListener.java +++ b/src/com/fuse/actions/bootstrap/AppBootstrapListener.java @@ -19,11 +19,15 @@ import com.fuse.dao.Assessment; import com.fuse.dao.HibHelper; import com.fuse.dao.Image; +import com.fuse.dao.ReportOptions; import com.fuse.dao.User; import com.fuse.dao.Vulnerability; import com.fuse.dao.DefaultVulnerability; import com.fuse.dao.Category; +import com.fuse.reporting.DocxPrecompiler; +import com.fuse.utils.FSUtils; import com.fuse.utils.LoggingConfig; +import com.fuse.utils.ReportImageScaler; @WebListener @@ -40,6 +44,7 @@ public void contextInitialized(ServletContextEvent sce) { createDefautlStatusIfNeeded(); fixAssessmentStatuses(); + prepareReportImageRenditions(); // Bootstrap method to create assessment with 500 vulnerabilities and 1000 images // Uncomment the following line to enable this bootstrap functionality @@ -106,98 +111,177 @@ private void fixAssessmentStatuses() { } } } - + + /** + * Backfill: prepares the report-ready rendition (see ReportImageScaler) + * for stored images that don't have one for the current width cap. New + * uploads are prepared inline at upload time; this covers images that + * existed before, plus every image after a FACTION_REPORT_IMAGE_MAX_WIDTH + * change. Runs on a background thread so startup isn't blocked; images + * are loaded and committed one at a time so heap usage stays flat. + * + * When the renditions are done, the same thread pre-compiles vuln HTML + * fields for open assessments (see DocxPrecompiler) — run second so the + * compiled XML embeds the freshly prepared renditions instead of + * re-downscaling originals. + */ + private void prepareReportImageRenditions() { + new Thread(() -> { + backfillImageRenditions(); + precompileOpenAssessmentVulns(); + }, "report-image-backfill").start(); + } + + private void backfillImageRenditions() { + try { + int maxWidth = ReportImageScaler.configuredMaxWidth(); + if (maxWidth <= 0) { + return; + } + List ids; + EntityManager em = HibHelper.getInstance().getEMF().createEntityManager(); + try { + // only images without a rendition for the current cap; + // fall back to a full scan if the OGM query translation + // rejects the null/inequality combination + try { + ids = em.createQuery( + "select i.id from Image i where i.reportWidth is null or i.reportWidth <> :w", + Long.class).setParameter("w", maxWidth).getResultList(); + } catch (Exception e) { + ids = em.createQuery("select i.id from Image i", Long.class).getResultList(); + } + } finally { + em.close(); + } + if (ids.isEmpty()) { + return; + } + System.out.println("[ReportImages] Preparing report renditions for up to " + + ids.size() + " images (width cap " + maxWidth + ")..."); + int prepared = 0; + long start = System.currentTimeMillis(); + for (Long id : ids) { + EntityManager iem = HibHelper.getInstance().getEMF().createEntityManager(); + try { + Image img = iem.find(Image.class, id); + if (img == null || !ReportImageScaler.prepareReportRendition(img)) { + continue; + } + HibHelper.getInstance().preJoin(); + iem.joinTransaction(); + iem.merge(img); + HibHelper.getInstance().commit(); + prepared++; + } catch (Exception e) { + System.err.println("[ReportImages] Error preparing image " + id + ": " + e.getMessage()); + } finally { + if (iem.isOpen()) { + iem.close(); + } + } + } + System.out.println("[ReportImages] Backfill complete: " + prepared + "/" + ids.size() + + " images prepared in " + (System.currentTimeMillis() - start) + "ms"); + } catch (Exception e) { + System.err.println("[ReportImages] Backfill failed: " + e.getMessage()); + e.printStackTrace(); + } + } + /** - * Bootstrap method to create an assessment with 500 vulnerabilities and 1000 images - * Uncomment this method to run it during application startup + * Pre-compiles HTML fields (desc/rec/details) into cached OOXML for + * all vulnerabilities in open assessments (see DocxPrecompiler). + * Vulnerabilities whose cache already matches their current content + * are skipped, so re-runs at every startup only pay the id query plus + * a per-vuln hash check — this is also how a CACHE_VERSION bump + * recompiles the whole install. New and updated vulnerabilities get + * their cache populated at save time by the save-path hooks. */ - private void createTestAssessmentWithVulnerabilitiesAndImages() { + private void precompileOpenAssessmentVulns() { EntityManager em = null; try { - System.out.println("Creating test assessment with 500 vulnerabilities and 1000 images..."); + System.out.println("[DocxPrecompiler] Checking pre-compiled caches for open assessments..."); em = HibHelper.getInstance().getEMF().createEntityManager(); - - // Create base64 encoded image from faction-logo.png - String base64Image = null; - try { - byte[] imageBytes = Files.readAllBytes(Paths.get("/Users/joshsummitt/Code/faction-all/free/faction/WebContent/faction-logo.png")); - base64Image = "data:image/png;base64," + Base64.getEncoder().encodeToString(imageBytes); - } catch (IOException e) { - System.err.println("Error reading faction logo file: " + e.getMessage()); + + // fetch report options once — font/CSS don't change per vuln + ReportOptions rpo = FSUtils.getOrCreateReportOptionsIfNotExist(em); + String font = rpo != null ? rpo.getFont() : "Calibri"; + String css = rpo != null ? rpo.getBodyCss() : ""; + String customCSS = css != null ? css : ""; + + // find all OPEN assessment IDs first — Hibernate OGM (MongoDB) + // does not support multi-entity JPQL joins, so we can't join + // Vulnerability and Assessment in one query + List openAssessmentIds = em.createQuery( + "select a.id from Assessment a " + + "where a.status is null or a.status <> 'Completed'", + Long.class).getResultList(); + if (openAssessmentIds.isEmpty()) { + System.out.println("[DocxPrecompiler] No open assessments found."); return; } - - // Create 1000 Images - List images = new ArrayList<>(); - for (int i = 0; i < 1000; i++) { - Image image = new Image(); - image.setBase64Image(base64Image); - image.setName("Test Image " + i); - image.setContentType("image/png"); - images.add(image); + + List vulnIds = new ArrayList<>(); + for (Long asmtId : openAssessmentIds) { + vulnIds.addAll(em.createQuery( + "select v.id from Vulnerability v where v.assessmentId = :aid", + Long.class) + .setParameter("aid", asmtId) + .getResultList()); } - - // Create Assessment - Assessment assessment = new Assessment(); - assessment.setName("Test Assessment with 500 Vulnerabilities and 1000 Images"); - assessment.setSummary("This is a test assessment created for development purposes"); - assessment.setRiskAnalysis("Risk analysis for test assessment"); - assessment.setStart(new Date()); - assessment.setEnd(new Date()); - - // Assign user with ID 2 as the assessor - User assessor = em.find(User.class, 2L); - if (assessor != null) { - List assessors = new ArrayList<>(); - assessors.add(assessor); - assessment.setAssessor(assessors); - } else { - System.err.println("User with ID 2 not found for assessor assignment"); + em.close(); + em = null; + if (vulnIds.isEmpty()) { + System.out.println("[DocxPrecompiler] No vulnerabilities found in open assessments."); + return; } - - // Set the images to the assessment - assessment.setImages(images); - - // For JTA transactions, use the TransactionManager from HibHelper - HibHelper.getInstance().preJoin(); - em.persist(assessment); - em.flush(); - - // Create 500 vulnerabilities with image links (but don't try to set them on the assessment directly) - // This avoids the collection management issue with cascade="all-delete-orphan" - int imgId=0; - for (int i = 0; i < 500; i++) { - Vulnerability vuln = new Vulnerability(); - vuln.setName("Test Vulnerability " + i); - vuln.setDescription("This is a test vulnerability description for vulnerability number " + i); - vuln.setRecommendation("Recommendation for vulnerability " + i); - vuln.setOverall(5L); // Set overall level to 5 (Critical) - vuln.setAssessmentId(assessment.getId()); - - // Create the details with image links - StringBuilder details = new StringBuilder(); - details.append("

Vulnerability details with images:

"); - - // Add image link using a placeholder that will be replaced with actual assessment ID - String imageId1 = assessment.getId().toString() + ":" + images.get(imgId++).getGuid(); - details.append("\"image.png\""); - String imageId2 = assessment.getId().toString() + ":" + images.get(imgId++).getGuid(); - details.append("\"image.png\""); - - vuln.setDescription(details.toString()); - - // Persist vulnerability directly rather than trying to manage the collection - em.persist(vuln); - assessment.getVulns().add(vuln); + + System.out.println("[DocxPrecompiler] Found " + vulnIds.size() + + " vulnerabilities in open assessments."); + + int totalCompiled = 0; + int totalSkipped = 0; + long startTime = System.currentTimeMillis(); + + for (Long vulnId : vulnIds) { + // fresh EM per vuln so a multi-MB cached payload never + // accumulates in a shared persistence context + EntityManager vulnEm = null; + try { + vulnEm = HibHelper.getInstance().getEMF().createEntityManager(); + Vulnerability v = vulnEm.find(Vulnerability.class, vulnId); + if (v == null) continue; + + // fetch the parent assessment so extensions can run + // during pre-compilation + Assessment asmt = v.getAssessmentId() > 0 + ? vulnEm.find(Assessment.class, v.getAssessmentId()) : null; + DocxPrecompiler pre = new DocxPrecompiler(font, customCSS, asmt); + if (pre.compile(v)) { + HibHelper.getInstance().preJoin(); + vulnEm.joinTransaction(); + vulnEm.merge(v); + HibHelper.getInstance().commit(); + totalCompiled++; + } else { + totalSkipped++; + } + } catch (Exception e) { + System.err.println("[DocxPrecompiler] Error compiling vuln " + + vulnId + ": " + e.getMessage()); + } finally { + if (vulnEm != null && vulnEm.isOpen()) { + vulnEm.close(); + } + } } - - em.merge(assessment); - HibHelper.getInstance().commit(); - - System.out.println("Successfully created test assessment with 500 vulnerabilities and 1000 images"); - + + System.out.println("[DocxPrecompiler] Migration complete: " + totalCompiled + + " compiled, " + totalSkipped + " skipped (already cached) in " + + (System.currentTimeMillis() - startTime) + "ms"); } catch (Exception e) { - System.err.println("Error creating test assessment: " + e.getMessage()); + System.err.println("[DocxPrecompiler] Migration failed: " + e.getMessage()); e.printStackTrace(); } finally { if (em != null && em.isOpen()) { @@ -205,4 +289,5 @@ private void createTestAssessmentWithVulnerabilitiesAndImages() { } } } + } \ No newline at end of file diff --git a/src/com/fuse/actions/images/UploadImage.java b/src/com/fuse/actions/images/UploadImage.java index f72dd889..20e2211f 100644 --- a/src/com/fuse/actions/images/UploadImage.java +++ b/src/com/fuse/actions/images/UploadImage.java @@ -16,6 +16,7 @@ import com.fuse.dao.Vulnerability; import com.fuse.dao.query.AssessmentQueries; import com.fuse.utils.ImageBorderUtil; +import com.fuse.utils.ReportImageScaler; @Namespace("/portal") public class UploadImage extends FSActionSupport { @@ -26,6 +27,9 @@ public class UploadImage extends FSActionSupport { public String uploadVulnImage() throws IOException { Image image = new Image(); image.setBase64Image(encodedImage); + // prepare the report-ready rendition once here so report + // generation never has to decode the full-size original + ReportImageScaler.prepareReportRendition(image); Assessment assessment = AssessmentQueries.getAssessment(em, getSessionUser(), assessmentId); if(assessment != null) { assessment.getImages().add(image); diff --git a/src/com/fuse/api/assessments.java b/src/com/fuse/api/assessments.java index c79035b5..3ca0c29d 100644 --- a/src/com/fuse/api/assessments.java +++ b/src/com/fuse/api/assessments.java @@ -55,12 +55,15 @@ import com.fuse.dao.Image; import com.fuse.dao.Note; import com.fuse.dao.PeerReview; +import com.fuse.dao.ReportOptions; import com.fuse.dao.RiskLevel; import com.fuse.dao.User; import com.fuse.dao.Vulnerability; import com.fuse.dao.query.AssessmentQueries; import com.fuse.dao.query.VulnerabilityQueries; import com.fuse.reporting.GenerateReport; +import com.fuse.reporting.DocxPrecompiler; +import com.fuse.reporting.DocxUtils; import com.fuse.servlets.EventStreamServlet; import com.fuse.tasks.ReportGenThread; import com.fuse.tasks.TaskQueueExecutor; @@ -2008,6 +2011,9 @@ public Response updateVulnerability( em.persist(vuln); HibHelper.getInstance().commit(); this.broadcastVulnUpdates(u, vuln); + // pre-compile HTML fields in the background so report + // generation can skip the expensive XHTML conversion + precompileInBackground(vuln, FSUtils.getOrCreateReportOptionsIfNotExist(em)); } return Response.status(200).entity(Support.SUCCESS).build(); @@ -2049,6 +2055,9 @@ public Response uploadImage( Image image = new Image(); image.setBase64Image(outlineImage(encodedImage)); + // prepare the report-ready rendition once here so report + // generation never has to decode the full-size original + com.fuse.utils.ReportImageScaler.prepareReportRendition(image); HibHelper.getInstance().preJoin(); em.joinTransaction(); @@ -2275,4 +2284,34 @@ private void sendBroadcastMessage(User user, Long vulnId, Long assessmentId, Str } } + // Spawns a background thread to pre-compile vuln HTML fields into + // cached OOXML, so report generation can skip the expensive + // XHTMLImporterImpl.convert() calls. If the precompile fails the cache + // stays stale and report-time falls back to live conversion. + private void precompileInBackground(Vulnerability vuln, ReportOptions RPO) { + if (vuln == null) return; + final Long vulnId = vuln.getId(); + final Long assessmentId = vuln.getAssessmentId(); + final String font = RPO != null ? RPO.getFont() : "Calibri"; + final String css = RPO != null ? RPO.getBodyCss() : ""; + final String fullCSS = (css != null ? css : ""); + + new Thread(() -> { + try { + EntityManager em = HibHelper.getInstance().getEMF().createEntityManager(); + try { + Vulnerability v = em.find(Vulnerability.class, vulnId); + Assessment assessment = assessmentId != null ? em.find(Assessment.class, assessmentId) : null; + if (v != null) { + DocxPrecompiler.precompileAndPersist(v, font, fullCSS, assessment); + } + } finally { + em.close(); + } + } catch (Exception e) { + e.printStackTrace(); + } + }, "docx-precompile-" + vulnId).start(); + } + } diff --git a/src/com/fuse/dao/Image.java b/src/com/fuse/dao/Image.java index ed189acd..38cc43da 100644 --- a/src/com/fuse/dao/Image.java +++ b/src/com/fuse/dao/Image.java @@ -26,6 +26,15 @@ public class Image { private String name; private String contentType; private String guid; + // Report-ready rendition: base64Image downscaled/re-encoded once at + // upload (or by the startup backfill) so report generation can embed + // it directly instead of decoding the full-size original for every + // report. null with a non-null reportWidth means the original already + // fits the cap; reportWidth records the width cap the rendition was + // prepared for, so a FACTION_REPORT_IMAGE_MAX_WIDTH change + // invalidates it. + private String reportImage; + private Integer reportWidth; public Image(){ UUID uuid = UUID.randomUUID(); @@ -63,6 +72,18 @@ public void setGuid(String guid) { UUID uuid = UUID.randomUUID(); this.guid = uuid.toString(); } + public String getReportImage() { + return reportImage; + } + public void setReportImage(String reportImage) { + this.reportImage = reportImage; + } + public Integer getReportWidth() { + return reportWidth; + } + public void setReportWidth(Integer reportWidth) { + this.reportWidth = reportWidth; + } diff --git a/src/com/fuse/dao/Vulnerability.java b/src/com/fuse/dao/Vulnerability.java index 785110e6..8442d8d3 100644 --- a/src/com/fuse/dao/Vulnerability.java +++ b/src/com/fuse/dao/Vulnerability.java @@ -78,7 +78,24 @@ public class Vulnerability { @ManyToOne private User detail_locked_by; private Date detail_lock_time; - + + // Pre-compiled OOXML for desc/rec/details, produced by DocxPrecompiler + // at vuln save time so report generation can skip the expensive + // XHTMLImporterImpl.convert() call. The payload is self-contained: + // images are embedded as data URIs and list numbering definitions + // travel with the XML (see DocxPrecompiler). The companion hash tracks + // the input content for invalidation; a version bump in the hash + // invalidates stale-format rows automatically. + private String cachedDescXml; + private String cachedRecXml; + private String cachedDetailsXml; + private String cachedDescHash; + private String cachedRecHash; + private String cachedDetailsHash; + // pre-compiled type-3 (HTML) custom fields, one marker-delimited entry + // per field with its own content hash (see DocxPrecompiler.findCfEntry) + private String cachedCfXml; + @Transient private Listlevels = new ArrayList(); @@ -362,9 +379,66 @@ public void setLevels(List levels) { public String getSectionPretty() { return this.section == null || this.section == ""? "Default" : this.section.replaceAll("_", " "); } - - - + + public String getCachedDescXml() { + return cachedDescXml; + } + + public void setCachedDescXml(String cachedDescXml) { + this.cachedDescXml = cachedDescXml; + } + + public String getCachedRecXml() { + return cachedRecXml; + } + + public void setCachedRecXml(String cachedRecXml) { + this.cachedRecXml = cachedRecXml; + } + + public String getCachedDetailsXml() { + return cachedDetailsXml; + } + + public void setCachedDetailsXml(String cachedDetailsXml) { + this.cachedDetailsXml = cachedDetailsXml; + } + + public String getCachedDescHash() { + return cachedDescHash; + } + + public void setCachedDescHash(String cachedDescHash) { + this.cachedDescHash = cachedDescHash; + } + + public String getCachedRecHash() { + return cachedRecHash; + } + + public void setCachedRecHash(String cachedRecHash) { + this.cachedRecHash = cachedRecHash; + } + + public String getCachedDetailsHash() { + return cachedDetailsHash; + } + + public void setCachedDetailsHash(String cachedDetailsHash) { + this.cachedDetailsHash = cachedDetailsHash; + } + + public String getCachedCfXml() { + return cachedCfXml; + } + + public void setCachedCfXml(String cachedCfXml) { + this.cachedCfXml = cachedCfXml; + } + + + + diff --git a/src/com/fuse/extenderapi/Extensions.java b/src/com/fuse/extenderapi/Extensions.java index 57103c45..b821afb1 100644 --- a/src/com/fuse/extenderapi/Extensions.java +++ b/src/com/fuse/extenderapi/Extensions.java @@ -225,6 +225,14 @@ private List cloneCustomFields(Vulnerability d public String updateReport(Assessment localAssessment, String reportText) { if (!this.isExtended()) return reportText; + // fast path: skip the expensive full-assessment clone when the text + // doesn't contain any ${ placeholder that an extension might handle. + // Extensions use ${faction-*} by convention; any extension-specific + // placeholder will contain "${". This avoids cloning 447 vulns + + // checklists + assessors for every field that has no extension work. + if (reportText == null || !reportText.contains("${")) { + return reportText; + } try { // Clone Assessment com.faction.elements.Assessment tmpAssessment = new com.faction.elements.Assessment(); diff --git a/src/com/fuse/reporting/DocxPrecompiler.java b/src/com/fuse/reporting/DocxPrecompiler.java new file mode 100644 index 00000000..0f3c0f91 --- /dev/null +++ b/src/com/fuse/reporting/DocxPrecompiler.java @@ -0,0 +1,678 @@ +package com.fuse.reporting; + +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.docx4j.XmlUtils; +import org.docx4j.convert.in.xhtml.XHTMLImporterImpl; +import org.docx4j.jaxb.Context; +import org.docx4j.openpackaging.exceptions.Docx4JException; +import org.docx4j.openpackaging.packages.WordprocessingMLPackage; +import org.docx4j.openpackaging.parts.WordprocessingML.BinaryPart; +import org.docx4j.openpackaging.parts.WordprocessingML.NumberingDefinitionsPart; +import org.docx4j.openpackaging.parts.relationships.RelationshipsPart; +import org.docx4j.relationships.Relationship; +import org.docx4j.wml.Numbering; +import org.docx4j.wml.RFonts; + +import com.fuse.dao.CustomField; +import com.fuse.dao.HibHelper; +import com.fuse.dao.Assessment; +import com.fuse.dao.Image; +import com.fuse.dao.Vulnerability; +import com.fuse.extenderapi.Extensions; +import com.fuse.utils.FSUtils; +import com.fuse.utils.LoggingConfig; +import com.fuse.utils.ReportImageScaler; + +import org.apache.commons.lang.StringUtils; + +import javax.persistence.EntityManager; + +/** + * Pre-compiles vulnerability HTML fields (desc/rec/details) into serialized + * OOXML at save time, so report generation can skip the expensive + * XHTMLImporterImpl.convert() call. + * + * The cached XML must be SELF-CONTAINED — a paragraph that references + * anything by id is worthless once it leaves the package the conversion + * ran in. Two kinds of references are made portable: + * + * Images: the importer attaches image parts to the scratch package and + * references them by rId. The image bytes are captured and embedded + * directly into the cached XML as data URIs (r:embed="data:image/..."); + * report-time code creates real parts in the report package and swaps in + * fresh rIds. + * + * Numbering: a list paragraph carries only , + * while what N MEANS (bullet vs decimal) lives in the package's + * numbering part. The v1 cache stored the bare numId and dropped the + * definitions — spliced into a report whose template owned those ids, + * bullet lists rendered as continued decimal numbers. Now the scratch + * package's entire numbering part is serialized into the cache payload + * (behind NUM_DEFS_MARKER) and the field XML's numId references are + * rewritten to FCT-NUM-<id> tokens. At report time + * DocxUtils.spliceCachedNumbering() re-creates every definition under + * freshly allocated ids and substitutes the tokens, so the template's + * own numbering is never referenced or disturbed. Each field gets its + * own scratch package so the captured numbering is exactly that field's. + * + * Assessment-level variables (${asmtName}, ${today}, etc.) are left as + * literal text inside nodes — the importer treats them as plain + * text. At report time, DocxUtils resolves them via string replacement + * on the cached XML, then unmarshals to JAXB nodes. + * + * Cache invalidation: the content hash (SHA-256 of version+font+content) + * is stored alongside the XML. At report time, if the hash doesn't match + * the current content, the cache is stale and wrapHTML falls back to + * live conversion. Bumping CACHE_VERSION invalidates every existing + * cache — including all v1-era rows still in the database. + */ +public class DocxPrecompiler { + + private static final Pattern IMAGE_LINK_PATTERN = Pattern.compile( + "]+src=[\"']getImage\\?id(=|=)[0-9]+:([^\"'\\s>]+)[\"'][^>]*>", + Pattern.CASE_INSENSITIVE); + + /** + * Separates the field's paragraph XML from the serialized numbering + * definitions in the cached payload. An XML comment can never appear + * inside marshalled OOXML, so the split is unambiguous. + */ + public static final String NUM_DEFS_MARKER = ""; + + private final String font; + private final String customCSS; + private final int maxImageWidth; + + // fresh per field (see compileField) — numbering definitions captured + // from it must belong to exactly one field + private WordprocessingMLPackage scratchPackage; + + // image rIds captured from the scratch package after conversion, + // keyed by importer rId (e.g. "rId3"); reset per field + private Map capturedImages = new HashMap<>(); + private Map capturedContentTypes = new HashMap<>(); + + // the parent assessment — needed to run report extensions at pre-compile + // time so their output is baked into the cached XML + private final Assessment assessment; + // report extension instance, lazily initialized + private Extensions reportExtension; + + public DocxPrecompiler(String font, String customCSS) { + this(font, customCSS, null); + } + + public DocxPrecompiler(String font, String customCSS, Assessment assessment) { + this.font = font == null ? "Calibri" : font; + this.customCSS = customCSS == null ? "" : customCSS; + this.maxImageWidth = ReportImageScaler.configuredMaxWidth(); + this.assessment = assessment; + } + + /** + * Pre-compiles all three HTML fields on a vulnerability and stores the + * results. Only fields whose content has changed (hash mismatch) are + * reconverted; unchanged fields keep their existing cache. + * + * @return true if any cache was updated + */ + public boolean compile(Vulnerability v) { + boolean updated = false; + + // resolve custom fields into the content before hashing — this is + // what getDescription/getRecommendation/getDetails do at report time + String desc = getDescription(v); + String rec = getRecommendation(v); + String details = v.getDetails() != null ? v.getDetails() : ""; + details = replaceAllCf(details, v); + + String descHash = hash(font, desc); + String recHash = hash(font, rec); + String detailsHash = hash(font, details); + + if (desc != null && !desc.isEmpty() && !descHash.equals(v.getCachedDescHash())) { + try { + String xml = compileField(desc, "desc"); + v.setCachedDescXml(xml); + v.setCachedDescHash(descHash); + updated = true; + } catch (Exception e) { + e.printStackTrace(); + // leave cache stale; report-time will fall back to live + v.setCachedDescXml(null); + v.setCachedDescHash(null); + } + } + + if (rec != null && !rec.isEmpty() && !recHash.equals(v.getCachedRecHash())) { + try { + String xml = compileField(rec, "rec"); + v.setCachedRecXml(xml); + v.setCachedRecHash(recHash); + updated = true; + } catch (Exception e) { + e.printStackTrace(); + v.setCachedRecXml(null); + v.setCachedRecHash(null); + } + } + + if (!details.isEmpty() && !detailsHash.equals(v.getCachedDetailsHash())) { + try { + String xml = compileField(details, "details"); + v.setCachedDetailsXml(xml); + v.setCachedDetailsHash(detailsHash); + updated = true; + } catch (Exception e) { + e.printStackTrace(); + v.setCachedDetailsXml(null); + v.setCachedDetailsHash(null); + } + } + + // type-3 (HTML) custom fields — entries with a matching hash are + // carried over, changed/new values are recompiled + try { + String rebuiltCf = compileCustomFields(v); + String existing = v.getCachedCfXml(); + if (rebuiltCf == null ? existing != null : !rebuiltCf.equals(existing)) { + v.setCachedCfXml(rebuiltCf); + updated = true; + } + } catch (Exception e) { + e.printStackTrace(); + v.setCachedCfXml(null); + } + + return updated; + } + + /** + * One cached entry per marker: {@code payload}. + * The payload is the same self-contained format compileField produces + * (paragraph XML, embedded images, numbering behind NUM_DEFS_MARKER). + */ + public static final String CF_ENTRY_PREFIX = "").append(payload); + } + return sb.length() == 0 ? null : sb.toString(); + } + + /** + * Finds the cached entry for a custom-field variable. + * + * @return {hash, payload}, or null when the variable has no entry + */ + public static String[] findCfEntry(String cachedCfXml, String variable) { + if (cachedCfXml == null || variable == null) { + return null; + } + int idx = 0; + while ((idx = cachedCfXml.indexOf(CF_ENTRY_PREFIX, idx)) != -1) { + int headEnd = cachedCfXml.indexOf("-->", idx); + if (headEnd < 0) { + return null; + } + String head = cachedCfXml.substring(idx + CF_ENTRY_PREFIX.length(), headEnd); + int nextEntry = cachedCfXml.indexOf(CF_ENTRY_PREFIX, headEnd); + int sp = head.lastIndexOf(' '); + if (sp > 0 && head.substring(0, sp).equals(variable)) { + String payload = cachedCfXml.substring(headEnd + 3, + nextEntry < 0 ? cachedCfXml.length() : nextEntry); + return new String[] { head.substring(sp + 1), payload }; + } + idx = nextEntry < 0 ? cachedCfXml.length() : nextEntry; + } + return null; + } + + /** + * Convenience method to pre-compile a vulnerability's HTML fields + * after it has been persisted. Looks up report options (font + CSS), + * runs the precompiler, and persists the cached XML back. + * + * Safe to call from save paths — if anything fails, the cache is + * simply left stale and report-time will fall back to live conversion. + */ + public static void precompileAndPersist(Vulnerability v, String font, String customCSS) { + precompileAndPersist(v, font, customCSS, null); + } + + public static void precompileAndPersist(Vulnerability v, String font, String customCSS, Assessment assessment) { + if (v == null) return; + try { + DocxPrecompiler pre = new DocxPrecompiler(font, customCSS, assessment); + if (pre.compile(v)) { + // persist the cached fields — use a fresh EM from the EMF + // to avoid closing the shared HibHelper singleton that + // report generation might be using concurrently + EntityManager em = HibHelper.getInstance().getEMF().createEntityManager(); + try { + HibHelper.getInstance().preJoin(); + em.joinTransaction(); + em.merge(v); + HibHelper.getInstance().commit(); + } finally { + em.close(); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * Hashes content the same way compile() does, so callers can check + * staleness without running the precompiler. + */ + public static String contentHash(String font, String content) { + return hash(font, content); + } + + // ===== per-field compilation ===== + + private String compileField(String content, String className) throws Docx4JException { + // Apply per-vuln-pre-compilable transformations: + // - \n →
+ // - blockquote → center.figure + // - jtidy (HTML cleanup) + // - misnested-list repair (ul directly inside ol, editor artifact) + // - image link resolution (getImage?id= → data URI) + // - inline image downscaling + // + // Do NOT apply: + // - assessment-level variables (leave ${asmtName} as literal text) + // - loopReplace (assessment-scoped) + + // fresh scratch per field: the numbering part captured below must + // contain exactly this field's list definitions and nothing else + try { + this.scratchPackage = WordprocessingMLPackage.createPackage(); + } catch (Exception e) { + throw new Docx4JException("scratch package creation failed", e); + } + this.capturedImages.clear(); + this.capturedContentTypes.clear(); + + content = content.replaceAll("\n", "
"); + content = content.replaceAll("


", ""); + content = content.replaceAll("
", "
"); + content = content.replaceAll("
", ""); + content = FSUtils.jtidy(content); + content = DocxUtils.hoistMisnestedLists(content); + + // run report extensions at pre-compile time so their output (e.g. + // injected charts, cross-references) is baked into the cached XML. + // Only runs when the content contains a ${ placeholder; the + // short-circuit inside updateReport() avoids the expensive clone. + if (assessment != null && content.contains("${")) { + if (reportExtension == null) { + reportExtension = new Extensions(HibHelper.getInstance().getEMF(), + Extensions.EventType.REPORT_MANAGER); + } + if (reportExtension.isExtended()) { + content = reportExtension.updateReport(assessment, content); + } + } + + // resolve getImage?id=... links to base64 data URIs + content = resolveImageLinks(content); + + // downscale oversized inline images + if (maxImageWidth > 0 && content.contains("data:image/")) { + content = downscaleInlineImages(content); + } + + // build full HTML page and convert + String html = htmlPageHead(customCSS) + "
" + content + "
"; + + XHTMLImporterImpl importer = newImporter(); + List converted = importer.convert(html, null); + captureImages(); + + // marshal to XML string, stripping declarations + String xml = marshalNodesToXml(converted); + + // replace scratch-package rId references with the actual image data + // URIs. At report time, DocxUtils creates real image parts in the + // report package and replaces the data URIs with valid rIds + xml = embedImageDataUris(xml); + + // capture the numbering definitions this conversion created and + // rewrite the paragraphs' numId references to portable tokens — + // see the class comment for why bare numIds must never be cached + NumberingDefinitionsPart ndp = scratchPackage.getMainDocumentPart().getNumberingDefinitionsPart(); + if (ndp != null && ndp.getContents() != null && !ndp.getContents().getNum().isEmpty()) { + Numbering numbering = ndp.getContents(); + for (Numbering.Num n : numbering.getNum()) { + xml = xml.replace("w:numId w:val=\"" + n.getNumId() + "\"", + "w:numId w:val=\"FCT-NUM-" + n.getNumId() + "\""); + } + xml = xml + NUM_DEFS_MARKER + XmlUtils.marshaltoString(numbering, true, false); + } + + return xml; + } + + // replaces r:embed="rIdN" with r:embed="data:image/..." using the + // captured image bytes. This makes the cached XML self-contained — + // no dependency on the scratch package's relationship table. + private String embedImageDataUris(String xml) { + if (capturedImages.isEmpty()) return xml; + for (Map.Entry entry : capturedImages.entrySet()) { + String rId = entry.getKey(); + byte[] bytes = entry.getValue(); + String contentType = capturedContentTypes.get(rId); + if (contentType == null) contentType = "image/png"; + String dataUri = "data:" + contentType + ";base64," + Base64.getEncoder().encodeToString(bytes); + // r:embed="rId3" → r:embed="data:image/png;base64,..." + xml = xml.replace("r:embed=\"" + rId + "\"", "r:embed=\"" + dataUri + "\""); + // also handle r:link (rare) + xml = xml.replace("r:link=\"" + rId + "\"", "r:link=\"" + dataUri + "\""); + } + return xml; + } + + // ===== image link resolution (getImage?id=...) ===== + + private String resolveImageLinks(String text) { + if (text == null || text.isEmpty() || !text.contains("getImage")) { + return text; + } + + // normalize img tags + text = text.replaceAll("(]*?)\\s*/?>(?!)", "$1>"); + text = text.replaceAll("


", ""); + + // remove undefined image links + text = text.replaceAll("]*src=[\"']getImage\\?id(=|=)undefined[\"'][^>]*>()?", ""); + + Pattern imagePattern = IMAGE_LINK_PATTERN; + Set referencedGuids = new HashSet<>(); + Matcher matcher = imagePattern.matcher(text); + while (matcher.find()) { + referencedGuids.add(matcher.group(2)); + } + if (referencedGuids.isEmpty()) { + return text; + } + + Map resolved = resolveImages(referencedGuids); + + StringBuffer result = new StringBuffer(); + matcher = imagePattern.matcher(text); + while (matcher.find()) { + String guid = matcher.group(2); + String base64Image = resolved.get(guid); + if (base64Image != null) { + matcher.appendReplacement(result, Matcher.quoteReplacement("")); + } else { + matcher.appendReplacement(result, ""); + } + } + matcher.appendTail(result); + return result.toString(); + } + + private Map resolveImages(Set guids) { + Map resolved = new HashMap<>(); + // use a fresh EM from the EMF, NOT the shared HibHelper.getEM() + // singleton — closing the singleton breaks any concurrent report + // generation that also uses it + EntityManager em = HibHelper.getInstance().getEMF().createEntityManager(); + try { + for (String guid : guids) { + try { + Image img = (Image) em.createQuery("SELECT i FROM Image i WHERE i.guid = :guid") + .setParameter("guid", guid).getSingleResult(); + if (img != null && img.getBase64Image() != null) { + resolved.put(guid, ReportImageScaler.reportUri(img, maxImageWidth)); + } + } catch (Exception e) { + // image not found + } + } + } finally { + em.close(); + } + return resolved; + } + + // ===== scratch package image capture ===== + + // After each convert() call, scan the scratch package's relationships for + // new image parts and capture their bytes so they can be embedded into + // the cached XML as data URIs. + private void captureImages() { + if (scratchPackage == null) return; + RelationshipsPart rels = scratchPackage.getMainDocumentPart().getRelationshipsPart(); + if (rels == null || rels.getRelationships() == null) return; + + for (Relationship r : rels.getRelationships().getRelationship()) { + if (r.getType() == null || !r.getType().endsWith("/image")) { + continue; + } + String rId = r.getId(); + if (capturedImages.containsKey(rId)) { + continue; + } + try { + org.docx4j.openpackaging.parts.Part part = rels.getPart(r); + if (part instanceof BinaryPart) { + BinaryPart bp = (BinaryPart) part; + byte[] bytes = bp.getBytes(); + capturedImages.put(rId, bytes); + capturedContentTypes.put(rId, bp.getContentType()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + // ===== helpers (mirrors DocxUtils) ===== + + private String htmlPageHead(String css) { + return "" + + "" + ""; + } + + private XHTMLImporterImpl newImporter() { + LoggingConfig.configureOpenHTMLTopDFLogging(); + XHTMLImporterImpl xhtml = new XHTMLImporterImpl(scratchPackage); + RFonts rfonts = Context.getWmlObjectFactory().createRFonts(); + rfonts.setAscii(this.font); + XHTMLImporterImpl.addFontMapping("Arial", rfonts); + XHTMLImporterImpl.addFontMapping("arial", rfonts); + return xhtml; + } + + private static String marshalNodesToXml(List nodes) { + StringBuilder sb = new StringBuilder(); + for (Object node : nodes) { + try { + String xml = XmlUtils.marshaltoString(node, false, false); + if (xml.startsWith(""); + if (close >= 0) { + xml = xml.substring(close + 2); + } + } + // normalize namespace prefixes to the standard w: prefix + // so splitTopLevelParagraphs can find at report time. + // docx4j's standalone marshaller may use arbitrary prefixes + // like which the splitter doesn't recognize. + xml = normalizeNamespaces(xml); + sb.append(xml); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + return sb.toString(); + } + + // Normalizes namespace prefixes to the standard OOXML prefixes so + // the report-time splitter can recognize elements. The docx4j + // marshaller assigns arbitrary prefixes (ns2, ns3, etc.) when + // marshalling standalone nodes outside a document context. + // + // Every replacement is anchored to XML structure — element names via + // the '<'/' uris = extractInlineDataUris(content); + if (uris.isEmpty()) return content; + StringBuilder sb = new StringBuilder(content); + for (String uri : uris) { + String downscaled = ReportImageScaler.downscaleDataUri(uri, maxImageWidth); + if (!downscaled.equals(uri)) { + int idx = sb.indexOf(uri); + if (idx >= 0) { + sb.replace(idx, idx + uri.length(), downscaled); + } + } + } + return sb.toString(); + } + + private static List extractInlineDataUris(String content) { + List uris = new ArrayList<>(); + int idx = 0; + while ((idx = content.indexOf("data:image/", idx)) != -1) { + if (idx == 0) { idx += 1; continue; } + char quote = content.charAt(idx - 1); + if (quote != '"' && quote != '\'') { idx += 1; continue; } + int end = content.indexOf(quote, idx); + if (end == -1) break; + uris.add(content.substring(idx, end)); + idx = end; + } + return uris; + } + + // Bump this when the cached XML format changes to invalidate all + // existing caches and force recompilation on the next migration/save. + // v7: numbering definitions captured + tokenized numIds — v6-era + // caches carried bare scratch-package numIds that collided with the + // report template's numbering (bullets rendered as decimal). + // v8: normalizeNamespaces text-corruption fix — v7 rows may have + // " :" in user text stored as " w:". + private static final String CACHE_VERSION = "v8"; + + // SHA-256 hash of version+font+content for cache invalidation + private static String hash(String font, String content) { + try { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + String input = CACHE_VERSION + "\0" + font + "\0" + (content == null ? "" : content); + byte[] digest = md.digest(input.getBytes("UTF-8")); + return Base64.getEncoder().encodeToString(digest); + } catch (Exception e) { + return "" + System.currentTimeMillis(); // fallback — always stale + } + } + + private static String replaceAllCf(String original, Vulnerability v) { + if (v.getCustomFields() != null) { + for (CustomField cf : v.getCustomFields()) { + try { + original = StringUtils.replace(original, + "${cf" + cf.getType().getVariable() + "}", + cf.getValue() == null ? "" : cf.getValue()); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + } + return original; + } + + // Mirrors DocxUtils.getDescription/getRecommendation — resolves + // default-vulnerability fallback and applies custom-field substitution. + private static String getDescription(Vulnerability v) { + String desc = ""; + if (v.getDescription() == null && v.getDefaultVuln() != null) { + desc = v.getDefaultVuln().getDescription(); + } else if (v.getDescription() != null) { + desc = v.getDescription(); + } + if (!desc.isEmpty()) { + return replaceAllCf(desc, v); + } + return desc; + } + + private static String getRecommendation(Vulnerability v) { + String rec = ""; + if (v.getRecommendation() == null && v.getDefaultVuln() != null) { + rec = v.getDefaultVuln().getRecommendation(); + } else if (v.getRecommendation() != null) { + rec = v.getRecommendation(); + } + if (!rec.isEmpty()) { + return replaceAllCf(rec, v); + } + return rec; + } +} diff --git a/src/com/fuse/reporting/DocxUtils.java b/src/com/fuse/reporting/DocxUtils.java index f5986d12..19c8c608 100644 --- a/src/com/fuse/reporting/DocxUtils.java +++ b/src/com/fuse/reporting/DocxUtils.java @@ -3,10 +3,6 @@ import java.awt.Color; import java.io.IOException; import java.io.StringWriter; -import java.math.BigInteger; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -14,10 +10,17 @@ import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; +import java.util.ListIterator; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -38,6 +41,7 @@ import org.docx4j.openpackaging.parts.WordprocessingML.FooterPart; import org.docx4j.openpackaging.parts.WordprocessingML.HeaderPart; import org.docx4j.openpackaging.parts.WordprocessingML.MainDocumentPart; +import org.docx4j.openpackaging.parts.WordprocessingML.NumberingDefinitionsPart; import org.docx4j.openpackaging.parts.relationships.RelationshipsPart; import org.docx4j.toc.TocException; import org.docx4j.toc.TocGenerator; @@ -48,6 +52,7 @@ import org.docx4j.wml.ContentAccessor; import org.docx4j.wml.Ftr; import org.docx4j.wml.Hdr; +import org.docx4j.wml.Numbering; import org.docx4j.wml.ObjectFactory; import org.docx4j.wml.P; import org.docx4j.wml.PPrBase.Ind; @@ -77,6 +82,7 @@ import com.fuse.utils.LoggingConfig; import com.fuse.utils.MethodProfiler; import com.fuse.utils.ProfileMethod; +import com.fuse.utils.ReportImageScaler; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -95,6 +101,48 @@ public class DocxUtils { private final WordprocessingMLPackage mlp; private String[] reportSections = new String[0]; private HashMap> nodeMap = new HashMap<>(); + // per-report caches: image lookups and the template's table list are + // reused across every vulnerability field instead of being rebuilt. + // The image cache is LRU-bounded so image-heavy assessments cannot + // hold every base64 payload in memory at once. Images are downscaled + // before caching, so the cap comfortably covers hundreds of images. + private static final long MAX_IMAGE_CACHE_CHARS = 64L * 1024 * 1024; // ~128MB of cached base64 + private LinkedHashMap imageCache = new LinkedHashMap<>(16, 0.75f, true); + private long imageCacheChars = 0; + private HashSet imagesNotFound = new HashSet<>(); + // oversized images are shrunk to this width before conversion — see + // ReportImageScaler; 0 disables + private final int maxImageWidth = ReportImageScaler.configuredMaxWidth(); + // downscaled images prepared by warmImageCache for THIS report's fields. + // Deliberately not size-capped: it holds at most the report's own + // (downscaled) images — which the document will embed anyway — and a cap + // here caused LRU thrashing where warmed entries were evicted and then + // re-fetched and re-downscaled serially, one field at a time + private HashMap warmedImages = new HashMap<>(); + // inline data-URI images downscaled ahead of time (in parallel) by + // warmImageCache, keyed by hash of the original URI + private ConcurrentHashMap inlineImagesDownscaled = new ConcurrentHashMap<>(); + // matches with either quote + // style and HTML-encoded '=' + private static final Pattern IMAGE_LINK_PATTERN = Pattern.compile( + "]+src=[\"']getImage\\?id(=|=)[0-9]+:([^\"'\\s>]+)[\"'][^>]*>", + Pattern.CASE_INSENSITIVE); + private List templateTables; + private Set processedTables = java.util.Collections.newSetFromMap(new IdentityHashMap()); + // assessment-level values that never change during a report run + private String assessorsLines = ""; + private String assessorsComma = ""; + private String assessorsBullets = ""; + private int[] riskCounts = new int[10]; + private int riskTotal = 0; + private String totalOpenVulns = "0"; + private String totalClosedVulns = "0"; + private static final ThreadLocal DATE_FORMAT = new ThreadLocal() { + @Override + protected SimpleDateFormat initialValue() { + return new SimpleDateFormat("MM/dd/yyyy"); + } + }; public DocxUtils(EntityManagerFactory entityManagerFactory, WordprocessingMLPackage mlp, Assessment assessment) { MethodProfiler.ProfileContext context = MethodProfiler.start("DocxUtils", "DocxUtils"); @@ -104,6 +152,7 @@ public DocxUtils(EntityManagerFactory entityManagerFactory, WordprocessingMLPack this.assessment = assessment; //this.outlineImages(); this.vulns = assessment.getVulns(); + this.precomputeAssessmentValues(); this.setupReportSections(entityManagerFactory); } finally { context.end(); @@ -119,12 +168,51 @@ public DocxUtils(WordprocessingMLPackage mlp, Assessment assessment) { this.assessment = assessment; // this.outlineImages(); this.vulns = assessment.getVulns(); + this.precomputeAssessmentValues(); this.setupReportSections(HibHelper.getInstance().getEMF()); } finally { context.end(); } } + // computed once per report instead of once per content field + private void precomputeAssessmentValues() { + if (this.assessment.getAssessor() != null) { + StringBuilder nl = new StringBuilder(); + StringBuilder comma = new StringBuilder(); + StringBuilder bullets = new StringBuilder("
    "); + boolean isfirst = true; + for (User hacker : this.assessment.getAssessor()) { + nl.append(hacker.getFname()).append(" ").append(hacker.getLname()).append("
    "); + comma.append(isfirst ? "" : ", ").append(hacker.getFname()).append(" ").append(hacker.getLname()); + bullets.append("
  • ").append(hacker.getFname()).append(" ").append(hacker.getLname()) + .append("
  • "); + isfirst = false; + } + bullets.append("
"); + this.assessorsLines = nl.toString(); + this.assessorsComma = comma.toString(); + this.assessorsBullets = bullets.toString(); + } + if (this.vulns != null) { + int open = 0; + int closed = 0; + for (Vulnerability v : this.vulns) { + if (v.getClosed() == null) { + open++; + } else { + closed++; + } + if (v.getOverall() == null || v.getOverall().intValue() == -1) + continue; + riskCounts[v.getOverall().intValue()]++; + riskTotal++; + } + this.totalOpenVulns = "" + open; + this.totalClosedVulns = "" + closed; + } + } + private void setupReportSections(EntityManagerFactory entityManagerFactory) { if (ReportFeatures.allowSections()) { EntityManager em = entityManagerFactory.createEntityManager(); @@ -143,30 +231,6 @@ private Boolean sectionExists(String section) { return Arrays.asList(this.reportSections).stream().anyMatch(reportSection -> reportSection.equals(section)); } - private boolean cellContains(Tc cell, String variable) { - for (Object obj : cell.getContent()) { - String xml = XmlUtils.marshaltoString(obj, false, false); - - if (xml.contains(variable)) { - return true; - } - } - return false; - } - - private Map setWidths(Tc cell, String variable, Map widths) { - if (cellContains(cell, "${" + variable + "}")) { - if (cell.getTcPr() != null && cell.getTcPr().getTcW() != null) { - BigInteger margin = BigInteger.valueOf(200); // TODO: This should not be hardcoded - widths.put(variable, cell.getTcPr().getTcW().getW().subtract(margin)); - } else { - widths.put(variable, BigInteger.valueOf(-1)); - } - } - return widths; - - } - @ProfileMethod("Search and Sort Vulns") private List getFilteredVulns(String section) { MethodProfiler.ProfileContext context = MethodProfiler.start("DocxUtils", "getFilteredVulns"); @@ -217,19 +281,21 @@ private void checkTables(String variable, String section, String customCSS) return; } - List tables = getAllElementFromObject(mlp.getMainDocumentPart(), Tbl.class); + // the template's tables are collected once per report — later + // passes would otherwise re-traverse a document that has grown + // by hundreds of generated rows + if (this.templateTables == null) { + this.templateTables = getAllElementFromObject(mlp.getMainDocumentPart(), Tbl.class); + } + List tables = this.templateTables; for (Object table : tables) { + // a table consumed by an earlier section had its template rows + // deleted and only grows with generated rows — skip rescans + if (this.processedTables.contains(table)) { + continue; + } MethodProfiler.ProfileContext context2 = MethodProfiler.start("DocxUtils", "sectionTest"); List paragraphs = getAllElementFromObject(table, P.class); - // This is to get a list of widths to ensure elements in tables behave - List cells = getAllElementFromObject(table, Tc.class); - Map widths = new HashMap<>(); - for (Object cell : cells) { - Tc tc = (Tc) cell; - widths = setWidths(tc, "desc", widths); - widths = setWidths(tc, "rec", widths); - widths = setWidths(tc, "details", widths); - } String tableVariable = "${" + variable + "}"; if (ReportFeatures.allowSections() && section != null && !section.isEmpty() && !section.equals("Default")) { @@ -239,6 +305,8 @@ private void checkTables(String variable, String section, String customCSS) if (txt == null) continue; + this.processedTables.add(table); + // Found a findings table // Get colorsMap if it exits; HashMap colorMap = new HashMap<>(); @@ -319,6 +387,16 @@ private void checkTables(String variable, String section, String customCSS) int count = 1; int sevIndex = 0; String prevSev = ""; + // HTML placeholders get a per-vulnerability suffix + // (e.g. ${rec:3}) so every field of the table can be + // converted in one batched pass instead of one importer + // call per field — the importer's fixed cost (CSS parse, + // font metrics, layout setup) dominated this path. + // Pre-compiled cache hits skip conversion entirely. + List fieldSpecs = new ArrayList<>(); + Set collectedKeys = new HashSet<>(); + HashMap> pendingNodes = new HashMap<>(); + List pendingRows = new ArrayList<>(); // {Tr row, List row keys} for (Vulnerability v : filteredVulns) { if (prevSev == v.getOverallStr()) { sevIndex++; @@ -326,9 +404,74 @@ private void checkTables(String variable, String section, String customCSS) prevSev = v.getOverallStr(); sevIndex = 1; } - // Change Colors if need be for (String xml : xmls) { - String nxml = replaceXml(xml, v, customFieldMap, colorMap, cellMap, null, count, sevIndex); + String nxml = replaceXml(xml, v, customFieldMap, colorMap, cellMap, null, count, sevIndex); + List rowKeys = new ArrayList<>(); + + if (xml.contains("${rec}")) { + String key = "${rec:" + count + "}"; + nxml = StringUtils.replace(nxml, "${rec}", key); + rowKeys.add(key); + if (collectedKeys.add(key)) { + List cached = wrapHTMLFromCache(v, "rec", count, customCSS); + if (cached != null) { + pendingNodes.put(key, cached); + } else { + fieldSpecs.add(new String[] { key, "rec", + this.replaceFigureVariables(getRecommendation(v), count) }); + } + } + } + if (xml.contains("${desc}")) { + String key = "${desc:" + count + "}"; + nxml = StringUtils.replace(nxml, "${desc}", key); + rowKeys.add(key); + if (collectedKeys.add(key)) { + List cached = wrapHTMLFromCache(v, "desc", count, customCSS); + if (cached != null) { + pendingNodes.put(key, cached); + } else { + fieldSpecs.add(new String[] { key, "desc", + this.replaceFigureVariables(getDescription(v), count) }); + } + } + } + if (xml.contains("${details}")) { + String key = "${details:" + count + "}"; + nxml = StringUtils.replace(nxml, "${details}", key); + rowKeys.add(key); + if (collectedKeys.add(key)) { + List cached = wrapHTMLFromCache(v, "details", count, customCSS); + if (cached != null) { + pendingNodes.put(key, cached); + } else { + fieldSpecs.add(new String[] { key, "details", + this.replaceFigureVariables(getDetails(v), count) }); + } + } + } + if (v.getCustomFields() != null) { + for (CustomField cf : v.getCustomFields()) { + if (cf.getType().getFieldType() == 3 + && xml.contains("${cf" + cf.getType().getVariable() + "}")) { + String key = "${cf" + cf.getType().getVariable() + ":" + count + "}"; + nxml = StringUtils.replace(nxml, + "${cf" + cf.getType().getVariable() + "}", key); + rowKeys.add(key); + if (collectedKeys.add(key)) { + List cfCached = wrapHTMLFromCache(v, + "cf:" + cf.getType().getVariable(), count, customCSS); + if (cfCached != null) { + pendingNodes.put(key, cfCached); + } else { + fieldSpecs.add(new String[] { key, cf.getType().getVariable(), + cf.getValue() == null ? "" : cf.getValue() }); + } + } + } + } + } + Tr newrow = (Tr) XmlUtils.unmarshalString(nxml); // Replace Hyperlinks @@ -340,42 +483,42 @@ private void checkTables(String variable, String section, String customCSS) } this.replaceHyperlink(newrow, "${cvssString link}", v.getCvssString()); - /* - * for(String match : cellMap.keySet()) changeColorOfCell(newrow, match, - * cellMap.get(match)); for(String match : colorMap.keySet()){ //TODO This - * should be deprecated with new method above changeColorOfText(newrow, match, - * colorMap.get(match)); } - */ - - HashMap> detailsMap = new HashMap<>(); - if (xml.contains("${rec}")) { - String rec = getRecommendation(v); - rec = this.replaceFigureVariables(rec, count); - detailsMap.put("${rec}", wrapHTML(rec, customCSS, "rec")); - } - if (xml.contains("${desc}")) { - String desc = getDescription(v); - desc = this.replaceFigureVariables(desc, count); - detailsMap.put("${desc}", wrapHTML(desc, customCSS, "desc")); - } - if (xml.contains("${details}")) { - String details = getDetails(v); - details = this.replaceFigureVariables(details, count); - detailsMap.put("${details}", wrapHTML(details, customCSS, "details")); + ((Tbl) table).getContent().add(newrow); + if (!rowKeys.isEmpty()) { + pendingRows.add(new Object[] { newrow, rowKeys }); } + } + count++; + } - if (v.getCustomFields() != null) { - for (CustomField cf : v.getCustomFields()) { - if (cf.getType().getFieldType() == 3) { - detailsMap.put("${cf" + cf.getType().getVariable() + "}", - wrapHTML(cf.getValue(), customCSS, cf.getType().getVariable())); + // batch-convert the cache misses, then fill each row. A key + // can appear in more than one row of the same vulnerability + // (multi-row loop templates); the first row consumes the + // converted nodes, later rows insert deep copies — the same + // JAXB nodes must never sit in the tree twice. + if (!pendingRows.isEmpty()) { + pendingNodes.putAll(convertFieldsBatched(fieldSpecs, customCSS)); + HashMap> insertedNodes = new HashMap<>(); + for (Object[] pending : pendingRows) { + Tr row = (Tr) pending[0]; + @SuppressWarnings("unchecked") + List rowKeys = (List) pending[1]; + HashMap> rowMap = new HashMap<>(); + for (String key : rowKeys) { + List nodes = pendingNodes.remove(key); + if (nodes == null) { + List used = insertedNodes.get(key); + if (used != null) { + nodes = deepCopyAll(used); } } + if (nodes != null) { + rowMap.put(key, nodes); + insertedNodes.put(key, nodes); + } } - replaceHTML(newrow, detailsMap); - ((Tbl) table).getContent().add(newrow); + replaceHTML(row, rowMap); } - count++; } // If no issues are discovered then we just blank out the table. if (filteredVulns == null || filteredVulns.size() == 0) { @@ -411,6 +554,24 @@ public WordprocessingMLPackage generateDocx(String customCSS) try { VariablePrepare.prepare(mlp); + // Assessment-level variable replacement runs BEFORE the findings + // are inserted: replacementText marshals the entire main document + // part, which is only the template here but 500+ findings big + // afterwards. Assessment variables inside finding content are + // resolved by the per-field paths — replacement() for live + // conversion, applyAssessmentVarsToXml for the pre-compiled + // cache — which cover the same variable set. + HashMap> map = new HashMap(); + + map.put("${summary1}", + this.wrapHTML(this.assessment.getSummary() == null ? "" : this.assessment.getSummary(), customCSS, + "summary1")); + map.put("${summary2}", + this.wrapHTML(this.assessment.getRiskAnalysis() == null ? "" : this.assessment.getRiskAnalysis(), + customCSS, "summary2")); + replaceHTML(mlp.getMainDocumentPart(), map, false); + replaceAssessment(customCSS); + // Convert all tables and match and replace values checkTables("vulnTable", "Default", customCSS); // look for findings areas {fiBegin/fiEnd} @@ -422,84 +583,778 @@ public WordprocessingMLPackage generateDocx(String customCSS) } } - HashMap> map = new HashMap(); - - map.put("${summary1}", - this.wrapHTML(this.assessment.getSummary() == null ? "" : this.assessment.getSummary(), customCSS, - "summary1")); - map.put("${summary2}", - this.wrapHTML(this.assessment.getRiskAnalysis() == null ? "" : this.assessment.getRiskAnalysis(), - customCSS, "summary2")); - replaceHTML(mlp.getMainDocumentPart(), map, false); - replaceAssessment(customCSS); insertPageBreaks(); updateDocWithExtensions(customCSS); + // release the per-report caches — the document is about to be + // serialized and converted downstream, which is when heap + // pressure peaks + nodeMap.clear(); + imageCache.clear(); + imageCacheChars = 0; + imagesNotFound.clear(); + warmedImages.clear(); + inlineImagesDownscaled.clear(); + embeddedImageRIdCache.clear(); + templateTables = null; + processedTables.clear(); + return mlp; } finally { context.end(); } } + // marker paragraph used to split one batched XHTML conversion back into + // its individual fields + private static final String FIELD_SPLIT_MARKER = "FCT-FIELD-SPLIT-7f3a91"; + // batch limits: enough fields per conversion to amortize the importer's + // fixed cost (page parse, CSS matcher build, layout setup), small enough + // to keep single-conversion memory bounded + private static final int MAX_CHUNK_FIELDS = 25; + private static final int MAX_CHUNK_CHARS = 2 * 1024 * 1024; + + // FACTION_REPORT_BATCH_CONVERT=false forces one conversion per field + // (the pre-batching behavior) — an isolation lever for rendering issues + private static boolean batchConvertEnabled() { + String conf = System.getProperty("FACTION_REPORT_BATCH_CONVERT"); + if (conf == null || conf.trim().isEmpty()) { + conf = System.getenv("FACTION_REPORT_BATCH_CONVERT"); + } + return conf == null || !conf.trim().equalsIgnoreCase("false"); + } + + private String preprocessHTMLContent(String content) { + if (!content.isEmpty()) { + content = replacement(content); + // fix extra spaces + content = content.replaceAll("\n", "
"); + // content = content.replaceAll("


", "

");//replace extra space + content = content.replaceAll("


", "");// replace extra space + content = content.replaceAll("
", "
"); + content = content.replaceAll("
", ""); + } + return content; + } + + private String htmlPageHead(String customCSS) { + return "" + + "" + ""; + } + + private XHTMLImporterImpl newImporter() { + MethodProfiler.ProfileContext context = MethodProfiler.start("DocxUtils", "newImporter"); + try{ + LoggingConfig.configureOpenHTMLTopDFLogging(); + XHTMLImporterImpl xhtml = new XHTMLImporterImpl(mlp); + RFonts rfonts = Context.getWmlObjectFactory().createRFonts(); + rfonts.setAscii(this.FONT); + XHTMLImporterImpl.addFontMapping("Arial", rfonts); + XHTMLImporterImpl.addFontMapping("arial", rfonts); + return xhtml; + } finally { + context.end(); + } + } + + // ===== pre-compiled cache fast path ===== + + /** + * Tries to serve wrapHTML from a pre-compiled cache stored on the + * Vulnerability at save time (by DocxPrecompiler). The cached XML + * has assessment-level ${...} variables as literal text inside + * nodes — this method resolves them via string replacement on the + * XML, then unmarshals to JAXB nodes. + * + * Falls back to live conversion (returns null sentinel) when: + * - cache is null or hash-stale (a CACHE_VERSION bump in the hash + * automatically invalidates every older-format row) + * - unmarshalling fails (malformed cache) + * - numbering tokens can't all be resolved + * + * Image data URIs in the cached XML become real image parts in the + * report package (resolveEmbeddedImages); cached numbering definitions + * are re-created under freshly allocated ids (spliceCachedNumbering) + * so list formats survive the move between packages. + * + * @param v the vulnerability whose cached field to use + * @param field "desc", "rec", or "details" + * @param count 1-based vulnerability index for figure numbering + * @return converted JAXB nodes, or null to fall back to live wrapHTML + */ + @ProfileMethod("wrapHTML from pre-compiled cache") + private List wrapHTMLFromCache(Vulnerability v, String field, int count, String customCSS) { + MethodProfiler.ProfileContext context = MethodProfiler.start("DocxUtils", "wrapHTMLFromCache"); + try { + String cachedXml; + String cachedHash; + String preContent; // content BEFORE replaceFigureVariables + + switch (field) { + case "desc": + cachedXml = v.getCachedDescXml(); + cachedHash = v.getCachedDescHash(); + preContent = getDescription(v); + break; + case "rec": + cachedXml = v.getCachedRecXml(); + cachedHash = v.getCachedRecHash(); + preContent = getRecommendation(v); + break; + case "details": + cachedXml = v.getCachedDetailsXml(); + cachedHash = v.getCachedDetailsHash(); + preContent = v.getDetails() != null ? v.getDetails() : ""; + preContent = replaceAllCf(preContent, v); + break; + default: + // "cf:" — a type-3 (HTML) custom field, cached + // as a marker-delimited entry with its own hash + if (!field.startsWith("cf:")) { + return null; + } + String cfVar = field.substring(3); + String cfValue = null; + if (v.getCustomFields() != null) { + for (CustomField cf : v.getCustomFields()) { + if (cf.getType() != null && cf.getType().getFieldType() == 3 + && cfVar.equals(cf.getType().getVariable())) { + cfValue = cf.getValue(); + break; + } + } + } + if (cfValue == null || cfValue.isEmpty()) { + return null; + } + String[] entry = DocxPrecompiler.findCfEntry(v.getCachedCfXml(), cfVar); + if (entry == null) { + return null; + } + cachedHash = entry[0]; + cachedXml = entry[1]; + preContent = cfValue; + break; + } + + if (cachedXml == null || cachedHash == null) { + return null; + } + + String expectedHash = DocxPrecompiler.contentHash(this.FONT, preContent); + if (!expectedHash.equals(cachedHash)) { + return null; + } + + // split the payload into field XML and its numbering definitions + String fieldXml = cachedXml; + String numberingXml = null; + int marker = cachedXml.indexOf(DocxPrecompiler.NUM_DEFS_MARKER); + if (marker >= 0) { + fieldXml = cachedXml.substring(0, marker); + numberingXml = cachedXml.substring(marker + DocxPrecompiler.NUM_DEFS_MARKER.length()); + } + + // safety net: numbering references without carried definitions + // can only come from a pre-v7 cache (bare scratch-package numIds + // that resolve against the template's numbering — the bug that + // rendered bullets as continued decimal numbers). The version + // bump already invalidates those via the hash; never trust one. + if (numberingXml == null && fieldXml.contains("w:numId")) { + return null; + } + + String xml = fieldXml; + + // Variable substitution runs dozens of full scans over the + // field XML (which can be megabytes with embedded images), so + // it is gated on the tokens actually being present — base64 + // payloads never contain "${" or "{[", so one indexOf each is + // a reliable test. Most fields have neither. + if (xml.indexOf("${") >= 0) { + // assessment-level ${asmtName} etc. survive as literal text + // inside nodes because the precompiler didn't resolve + // them; figure variables: ${Figure#.X} → Figure count.X + xml = applyAssessmentVarsToXml(xml); + xml = replaceFigureVariables(xml, count); + } + if (xml.indexOf("{[asmt") >= 0) { + // {[asmtSEVERITY]} ranked lists (assessment-scoped) + xml = loopReplace(xml); + } + + // resolve embedded image data URIs to real image parts in the + // report package. The precompiler embedded "data:image/..." as + // the r:embed value — we create BinaryParts and replace with + // valid rIds so Word can display the images + xml = resolveEmbeddedImages(xml); + + // re-create the field's numbering definitions in the report + // package and swap the FCT-NUM tokens for the fresh ids + if (numberingXml != null) { + xml = spliceCachedNumbering(xml, numberingXml); + if (xml == null || xml.contains("FCT-NUM-")) { + return null; // unresolved token — fall back to live + } + } + + // unmarshal the XML string back to JAXB nodes. The cached XML + // is a concatenation of elements — unmarshal each one + // individually to avoid deep JAXB call stacks on large snippets + List nodes = new ArrayList<>(); + try { + List snippets = splitTopLevelParagraphs(xml); + if (snippets.isEmpty()) { + // the cached XML doesn't contain recognizable + // elements — fall back to live conversion rather than + // returning an empty list. + return null; + } + for (String snippet : snippets) { + Object node = XmlUtils.unmarshalString(snippet); + nodes.add(node); + } + } catch (Exception e) { + e.printStackTrace(); + return null; // fall back to live + } + + if (nodes.isEmpty()) { + return null; // nothing unmarshalled — fall back to live + } + + // returned directly, not through nodeMap: deepCopyAll is a + // marshal+unmarshal round trip per node, so storing a shared + // copy and cloning it costs more than the unmarshal above — + // duplicate content just deserializes again. This also keeps + // numbering instances per-field (sharing nodes across fields + // would make identical numbered lists continue counting). + return nodes; + } finally { + context.end(); + } + } + + /** + * Re-creates a cached field's numbering definitions inside the report + * package and rewrites the field XML's FCT-NUM tokens to the freshly + * allocated ids. + * + * Every cached abstractNum/num gets its own new instance — never + * deduplicated or shared. The importer creates one abstractNum per + * list, and that per-list identity is exactly what makes numbered + * lists restart at 1 for each finding; sharing an abstract across + * instances would make Word continue the count across findings. + * + * @return the field XML with tokens substituted, or null on failure + */ + private String spliceCachedNumbering(String xml, String numberingXml) { + try { + Numbering cached = (Numbering) XmlUtils.unmarshalString(numberingXml); + NumberingDefinitionsPart ndp = mlp.getMainDocumentPart().getNumberingDefinitionsPart(); + if (ndp == null) { + ndp = new NumberingDefinitionsPart(); + ndp.setJaxbElement(Context.getWmlObjectFactory().createNumbering()); + mlp.getMainDocumentPart().addTargetPart(ndp); + } + Numbering target = ndp.getContents(); + + long maxAbs = 0; + for (Numbering.AbstractNum a : target.getAbstractNum()) { + if (a.getAbstractNumId() != null) { + maxAbs = Math.max(maxAbs, a.getAbstractNumId().longValue()); + } + } + long maxNum = 0; + for (Numbering.Num n : target.getNum()) { + if (n.getNumId() != null) { + maxNum = Math.max(maxNum, n.getNumId().longValue()); + } + } + + Map absMap = new HashMap<>(); + for (Numbering.AbstractNum a : cached.getAbstractNum()) { + Numbering.AbstractNum copy = XmlUtils.deepCopy(a); + java.math.BigInteger fresh = java.math.BigInteger.valueOf(++maxAbs); + absMap.put(a.getAbstractNumId(), fresh); + copy.setAbstractNumId(fresh); + target.getAbstractNum().add(copy); + } + for (Numbering.Num n : cached.getNum()) { + Numbering.Num copy = XmlUtils.deepCopy(n); + java.math.BigInteger fresh = java.math.BigInteger.valueOf(++maxNum); + copy.setNumId(fresh); + if (copy.getAbstractNumId() != null + && absMap.containsKey(copy.getAbstractNumId().getVal())) { + copy.getAbstractNumId().setVal(absMap.get(copy.getAbstractNumId().getVal())); + } + target.getNum().add(copy); + xml = xml.replace("w:val=\"FCT-NUM-" + n.getNumId() + "\"", "w:val=\"" + fresh + "\""); + } + return xml; + } catch (Exception e) { + e.printStackTrace(); + return null; + } + } + + // applies assessment-level ${...} variable substitutions on raw XML + // text. The cached XML has these as literal text inside nodes. + // CDATA-wrapping matches what replacementText() does for the document- + // wide pass. + private String applyAssessmentVarsToXml(String xml) { + xml = StringUtils.replace(xml, "${asmtName}", + CData(this.assessment.getName() == null ? "" : this.assessment.getName())); + xml = StringUtils.replace(xml, "${asmtId}", + CData(this.assessment.getId() == null ? "" : "" + this.assessment.getId())); + xml = StringUtils.replace(xml, "${asmtAppId}", CData("" + this.assessment.getAppId())); + xml = StringUtils.replace(xml, "${asmtAssessor}", CData(this.assessment.getAssessor() == null ? "" + : (this.assessment.getAssessor().get(0).getFname() + " " + + this.assessment.getAssessor().get(0).getLname()))); + xml = StringUtils.replace(xml, "${asmtAssessor_Email}", + CData(this.assessment.getAssessor() == null ? "" : (this.assessment.getAssessor().get(0).getEmail()))); + xml = StringUtils.replace(xml, "${asmtAssessors_Lines}", + CData(this.assessment.getAssessor() == null ? "" : this.assessorsLines)); + xml = StringUtils.replace(xml, "${asmtAssessors_Comma}", + CData(this.assessment.getAssessor() == null ? "" : this.assessorsComma)); + xml = StringUtils.replace(xml, "${asmtAssessors_Bullets}", + CData(this.assessment.getAssessor() == null ? "" : this.assessorsBullets)); + xml = StringUtils.replace(xml, "${remediation}", CData(this.assessment.getRemediation() == null ? "" + : (this.assessment.getRemediation().getFname() + " " + this.assessment.getRemediation().getLname()))); + xml = StringUtils.replace(xml, "${asmtTeam}", CData(this.assessment.getAssessor() == null ? "" + : this.assessment.getAssessor().get(0).getTeam() == null ? "" + : this.assessment.getAssessor().get(0).getTeam().getTeamName().trim())); + xml = StringUtils.replace(xml, "${asmtType}", + CData(this.assessment.getType() == null ? "" : this.assessment.getType().getType().trim())); + xml = replaceDateVariable(xml, "today", new Date()); + xml = replaceDateVariable(xml, "asmtStart", this.assessment.getStart()); + xml = replaceDateVariable(xml, "asmtEnd", this.assessment.getEnd()); + xml = StringUtils.replace(xml, "${asmtAccessKey}", CData(this.assessment.getGuid())); + xml = StringUtils.replace(xml, "${totalOpenVulns}", CData(this.totalOpenVulns)); + xml = StringUtils.replace(xml, "${totalClosedVulns}", CData(this.totalClosedVulns)); + // risk counts + if (this.vulns != null) { + for (int i = 0; i < 10; i++) { + xml = StringUtils.replace(xml, "${riskCount" + i + "}", CData("" + riskCounts[i])); + } + xml = StringUtils.replace(xml, "${riskTotal}", CData("" + riskTotal)); + } + // assessment-level text custom fields — with replaceAssessment now + // running before the findings are inserted, ${cfX} tokens inside + // finding content must be resolved here + if (this.assessment.getCustomFields() != null) { + for (CustomField cf : this.assessment.getCustomFields()) { + if (cf.getType() != null && cf.getType().getFieldType() < 3) { + xml = StringUtils.replace(xml, "${cf" + cf.getType().getVariable() + "}", + CData(cf.getValue() == null ? "" : cf.getValue())); + } + } + } + return xml; + } + + // per-report dedup: data URI → rId for images already created in the + // report package. The same screenshot embedded in multiple vulns + // (or desc+details of the same vuln) gets one image part, not many. + private Map embeddedImageRIdCache = new HashMap<>(); + + // Pattern to find r:embed="data:image/..." or r:link="data:image/..." + // in cached XML. The precompiler embedded the full data URI as the + // embed value so the cached XML is self-contained. + private static final Pattern EMBEDDED_IMAGE_PATTERN = Pattern.compile( + "(r:embed|r:link)=\"(data:image/[^\"]+)\""); + + /** + * Scans cached XML for embedded image data URIs (placed by the + * precompiler), creates real image parts in the report package, and + * replaces the data URIs with valid rId references. Deduplicates + * identical images across vulns/fields within a single report. + */ + private String resolveEmbeddedImages(String xml) { + if (!xml.contains("data:image/")) return xml; + + Matcher m = EMBEDDED_IMAGE_PATTERN.matcher(xml); + StringBuffer sb = new StringBuffer(); + while (m.find()) { + String attr = m.group(1); + String dataUri = m.group(2); + String rId = embeddedImageRIdCache.get(dataUri); + if (rId == null) { + try { + // decode the data URI and create an image part in the + // report package + String[] uriParts = dataUri.substring(5).split(",", 2); + byte[] bytes = Base64.getDecoder().decode(uriParts[1]); + String mime = uriParts[0].split(";")[0]; + // known formats get the part created directly — the + // probing path below re-parses every image to determine + // format and dimensions, which the cached drawing XML + // already carries from precompile time + rId = addEmbeddedImagePart(bytes, mime); + if (rId == null) { + // unknown format: let docx4j probe it + org.docx4j.openpackaging.parts.WordprocessingML.BinaryPartAbstractImage imagePart = + org.docx4j.openpackaging.parts.WordprocessingML.BinaryPartAbstractImage + .createImagePart(mlp, bytes); + org.docx4j.relationships.Relationship rel = imagePart.getRelLast(); + if (rel != null) { + rId = rel.getId(); + } + if (rId == null) { + // fallback: find the rId from the main document + // part's relationships to this image part + rId = mlp.getMainDocumentPart().getRelationshipsPart() + .getRelationships().getRelationship().stream() + .filter(r -> r.getTarget().contains(imagePart.getPartName().getName())) + .map(org.docx4j.relationships.Relationship::getId) + .reduce((first, second) -> second) + .orElse(null); + } + } + if (rId != null) { + embeddedImageRIdCache.put(dataUri, rId); + } else { + // couldn't find the rId — remove the reference + m.appendReplacement(sb, Matcher.quoteReplacement(attr + "=\"\"")); + continue; + } + } catch (Exception e) { + e.printStackTrace(); + m.appendReplacement(sb, Matcher.quoteReplacement(attr + "=\"\"")); + continue; + } + } + m.appendReplacement(sb, Matcher.quoteReplacement(attr + "=\"" + rId + "\"")); + } + m.appendTail(sb); + return sb.toString(); + } + + // sequence for unique media part names; "fctimage" avoids clashing with + // the template's own image1.png-style names + private int embeddedImagePartCounter = 0; + + // Creates an image part for a known format without docx4j's + // createImagePart() probe (which decodes every image to determine + // format and dimensions). Returns the relationship id, or null for + // formats the caller should hand to the probing path. + private String addEmbeddedImagePart(byte[] bytes, String mime) { + try { + org.docx4j.openpackaging.parts.WordprocessingML.BinaryPartAbstractImage part; + if (mime.contains("png")) { + part = new org.docx4j.openpackaging.parts.WordprocessingML.ImagePngPart( + new org.docx4j.openpackaging.parts.PartName( + "/word/media/fctimage" + (++embeddedImagePartCounter) + ".png")); + } else if (mime.contains("jpeg") || mime.contains("jpg")) { + part = new org.docx4j.openpackaging.parts.WordprocessingML.ImageJpegPart( + new org.docx4j.openpackaging.parts.PartName( + "/word/media/fctimage" + (++embeddedImagePartCounter) + ".jpeg")); + } else if (mime.contains("gif")) { + part = new org.docx4j.openpackaging.parts.WordprocessingML.ImageGifPart( + new org.docx4j.openpackaging.parts.PartName( + "/word/media/fctimage" + (++embeddedImagePartCounter) + ".gif")); + } else { + return null; + } + part.setBinaryData(bytes); + org.docx4j.relationships.Relationship rel = mlp.getMainDocumentPart().addTargetPart(part); + return rel == null ? null : rel.getId(); + } catch (Exception e) { + e.printStackTrace(); + return null; + } + } + + // splits a concatenated XML string into individual top-level paragraph + // elements, so each can be unmarshaled independently. This avoids deep + // JAXB call stacks when unmarshaling large snippets. + // + // The docx4j marshaller may use the standard prefix when + // marshalling inside a document context, or an arbitrary namespace + // prefix (e.g. ) when marshalling a standalone node. This + // method handles both by falling back to a regex-based split if the + // standard prefix scan finds nothing. + private static List splitTopLevelParagraphs(String xml) { + List parts = new ArrayList<>(); + int scan = 0; + while (scan < xml.length()) { + int pStart = xml.indexOf("", scan); + int best; + if (pStart < 0) { + best = pStartPlain; + } else if (pStartPlain < 0) { + best = pStart; + } else { + best = Math.min(pStart, pStartPlain); + } + if (best < 0) { + break; + } + // find matching with depth tracking for nested w:p + int depth = 0; + int i = best + 4; + int end = -1; + while (i < xml.length()) { + int nextOpen = xml.indexOf("", i); + int nextClose = xml.indexOf("", i); + int bestOpen; + if (nextOpen < 0) { + bestOpen = nextOpenPlain; + } else if (nextOpenPlain < 0) { + bestOpen = nextOpen; + } else { + bestOpen = Math.min(nextOpen, nextOpenPlain); + } + if (nextClose < 0) break; + if (bestOpen >= 0 && bestOpen < nextClose) { + depth++; + i = bestOpen + 5; + } else { + if (depth == 0) { + end = nextClose + "".length(); + break; + } + depth--; + i = nextClose + "".length(); + } + } + if (end < 0) break; + String snippet = xml.substring(best, end); + if (!snippet.contains("xmlns:w=")) { + snippet = snippet.replaceFirst(" elements were found, the marshaller may + // have used a different namespace prefix (e.g. ). Try a + // regex-based split on any element whose local name is "p". + if (parts.isEmpty() && xml.contains(":p") && xml.contains(":p>")) { + Pattern altPPattern = Pattern.compile( + "<\\w+:p[ >].*?", + Pattern.DOTALL); + Matcher m = altPPattern.matcher(xml); + while (m.find()) { + String snippet = m.group(); + // normalize the namespace prefix to w: so XmlUtils can + // unmarshal it in the standard wordprocessingml context — + // anchored to tag delimiters so text content is never touched + String prefix = snippet.substring(1, snippet.indexOf(":p")); + snippet = snippet.replace("<" + prefix + ":p", " wrapHTML(String content, String customCSS, String className) throws Docx4JException{ MethodProfiler.ProfileContext context = MethodProfiler.start("DocxUtils", "wrapHTML"); try { - + if (className == null) { className = ""; } - if(!content.isEmpty()) { - content = replacement(content); - // fix extra spaces - content = content.replaceAll("\n", "
"); - // content = content.replaceAll("


", "

");//replace extra space - content = content.replaceAll("


", "");// replace extra space - content = content.replaceAll("
", "
"); - content = content.replaceAll("
", ""); - } - LoggingConfig.configureOpenHTMLTopDFLogging(); - String html = "" - + "" + "
" - + content + "
"; - - String md5 = toMD5(html); - if (nodeMap.containsKey(md5)) { - return nodeMap.get(md5); - } else { - XHTMLImporterImpl xhtml = new XHTMLImporterImpl(mlp); - RFonts rfonts = Context.getWmlObjectFactory().createRFonts(); - rfonts.setAscii(this.FONT); - XHTMLImporterImpl.addFontMapping("Arial", rfonts); - XHTMLImporterImpl.addFontMapping("arial", rfonts); + + // cache on the raw input: identical content converts to identical + // nodes, and skipping straight past replacement()/jtidy/conversion + // is the whole win. Copies are returned because the same JAXB + // nodes must not be inserted into the document tree twice. + String cacheKey = className + " " + content; + List cached = nodeMap.get(cacheKey); + if (cached != null) { + return deepCopyAll(cached); + } + + content = preprocessHTMLContent(content); + String htmlPrefix = htmlPageHead(customCSS) + "
"; + String htmlSuffix = "
"; + String html = htmlPrefix + content + htmlSuffix; + + XHTMLImporterImpl xhtml = newImporter(); + MethodProfiler.ProfileContext convertContext = MethodProfiler.start("DocxUtils", "xhtmlConvert"); + try { + List converted = xhtml.convert(html, null); + nodeMap.put(cacheKey, converted); + return converted; + // jtidy is expensive, so the content is only repaired when + // the importer actually rejects it + }catch(Docx4JException ex){ + // tidy just the fragment and rebuild the page around it — + // jtidy outputs body-only, so tidying the whole page would + // drop the " + ""; + } + + private XHTMLImporterImpl newImporter() { + LoggingConfig.configureOpenHTMLTopDFLogging(); + XHTMLImporterImpl xhtml = new XHTMLImporterImpl(this.scratchPackage); + RFonts rfonts = Context.getWmlObjectFactory().createRFonts(); + rfonts.setAscii(this.FONT); + XHTMLImporterImpl.addFontMapping("Arial", rfonts); + XHTMLImporterImpl.addFontMapping("arial", rfonts); + return xhtml; + } + + private boolean isSplitMarker(Object node) { + Object unwrapped = XmlUtils.unwrap(node); + if (!(unwrapped instanceof P)) + return false; + StringWriter text = new StringWriter(); + try { + TextUtils.extractText(unwrapped, text); + } catch (Exception ex) { + return false; + } + return FIELD_SPLIT_MARKER.equals(text.toString().trim()); + } + + // ===== document-wide simple variables ===== + + /** + * Single-pass replacement of ${asmtName}, ${asmtId}, ${today}, + * ${asmtStart}, ${asmtEnd}, risk counts, etc. on a piece of document + * XML. Also handles the CDATA wrapping that the original DocxUtils did + * via marshalToString + replaceAll(_, _, true). + */ + private String applyDocumentWideVariables(String xml, String customCSS) { + SimpleDateFormat formatter = DATE_FORMAT.get(); + + Map dateMap = new LinkedHashMap<>(); + dateMap.put("today", new Date()); + dateMap.put("asmtStart", this.assessment.getStart()); + dateMap.put("asmtEnd", this.assessment.getEnd()); + for (Map.Entry e : dateMap.entrySet()) { + xml = replaceDateVariable(xml, e.getKey(), e.getValue()); + } + + xml = StringUtils.replace(xml, "${asmtName}", + CData(this.assessment.getName() == null ? "" : this.assessment.getName())); + xml = StringUtils.replace(xml, "${asmtId}", + CData(this.assessment.getId() == null ? "" : "" + this.assessment.getId())); + xml = StringUtils.replace(xml, "${asmtAppId}", CData("" + this.assessment.getAppId())); + xml = StringUtils.replace(xml, "${asmtAssessor}", CData(this.assessment.getAssessor() == null ? "" + : (this.assessment.getAssessor().get(0).getFname() + " " + + this.assessment.getAssessor().get(0).getLname()))); + xml = StringUtils.replace(xml, "${asmtAssessor_Email}", + CData(this.assessment.getAssessor() == null ? "" : (this.assessment.getAssessor().get(0).getEmail()))); + xml = StringUtils.replace(xml, "${asmtAssessors_Lines}", + CData(this.assessment.getAssessor() == null ? "" : this.assessorsLines)); + xml = StringUtils.replace(xml, "${asmtAssessors_Comma}", + CData(this.assessment.getAssessor() == null ? "" : this.assessorsComma)); + xml = StringUtils.replace(xml, "${asmtAssessors_Bullets}", + CData(this.assessment.getAssessor() == null ? "" : this.assessorsBullets)); + xml = StringUtils.replace(xml, "${remediation}", CData(this.assessment.getRemediation() == null ? "" + : (this.assessment.getRemediation().getFname() + " " + this.assessment.getRemediation().getLname()))); + xml = StringUtils.replace(xml, "${asmtTeam}", CData(this.assessment.getAssessor() == null ? "" + : this.assessment.getAssessor().get(0).getTeam() == null ? "" + : this.assessment.getAssessor().get(0).getTeam().getTeamName().trim())); + xml = StringUtils.replace(xml, "${asmtType}", + CData(this.assessment.getType() == null ? "" : this.assessment.getType().getType().trim())); + xml = StringUtils.replace(xml, "${asmtAccessKey}", CData(this.assessment.getGuid())); + xml = StringUtils.replace(xml, "${totalOpenVulns}", CData(this.totalOpenVulns)); + xml = StringUtils.replace(xml, "${totalClosedVulns}", CData(this.totalClosedVulns)); + + // risk counts + if (this.vulns != null) { + for (int i = 0; i < 10; i++) { + xml = StringUtils.replace(xml, "${riskCount" + i + "}", CData("" + riskCounts[i])); + } + xml = StringUtils.replace(xml, "${riskTotal}", CData("" + riskTotal)); + } + + // assessment-level custom fields (text and HTML) + if (this.assessment.getCustomFields() != null) { + for (CustomField cf : this.assessment.getCustomFields()) { + if (cf.getType().getFieldType() < 3) { + xml = StringUtils.replace(xml, "${cf" + cf.getType().getVariable() + "}", + CData(cf.getValue() == null ? "" : cf.getValue())); + } else if (cf.getType().getFieldType() == 3) { + String snippet = wrapHtmlAsXml(cf.getValue() == null ? "" : cf.getValue(), customCSS, + cf.getType().getVariable()); + // the placeholder lives inside a - replace the + // entire paragraph + String placeholder = "${cf" + cf.getType().getVariable() + "}"; + xml = replaceParagraphPlaceholders(xml, placeholder, snippet); + } + } + } + + // run report extensions + if (this.reportExtension.isExtended()) { + xml = this.reportExtension.updateReport(this.assessment, xml); + } + + // {[asmtVARNAME]} ranked lists + xml = loopReplace(xml); + + return xml; + } + + // Replaces every whose only text content is exactly placeholder + // with replacementXml. + private static String replaceParagraphPlaceholders(String xml, String placeholder, String replacementXml) { + StringBuilder out = new StringBuilder(); + int scan = 0; + while (true) { + int idx = xml.indexOf(placeholder, scan); + if (idx < 0) { + out.append(xml, scan, xml.length()); + break; + } + int[] bounds = findEnclosingParagraphBounds(xml, idx); + if (bounds == null) { + out.append(xml, scan, idx + placeholder.length()); + scan = idx + placeholder.length(); + continue; + } + out.append(xml, scan, bounds[0]); + out.append(replacementXml); + scan = bounds[1]; + } + return out.toString(); + } + + private String loopReplace(String content) { + for (int i = 9; i >= 0; i--) { + content = innerLoop(content, i); + } + return content; + } + + private String innerLoop(String content, int rank) { + Vulnerability tmp = new Vulnerability(); + String Var = tmp.vulnStr(new Long(rank)).toUpperCase(); + if (content.contains("{[asmt" + Var + "]}")) { + String html = "
    \r\n"; + boolean isSomething = false; + for (Vulnerability v : this.vulns) { + if (v.getOverall() == rank) { + isSomething = true; + html += "
  1. " + v.getName() + "
  2. "; + } + } + html += "
"; + if (!isSomething) { + html = "No vulnerabilities found at this severity. "; + } + content = content.replaceAll("\\{\\[assessment\\." + Var + "\\]\\}", html); + } + return content; + } + + // ===== rels.xml rebuild ===== + + /** + * Adds the output image relationships we accumulated during HTML + * conversion to the existing word/_rels/document.xml.rels content. + * Template relationships are preserved verbatim. + */ + private String rebuildRelsXml(String relsXml) { + if (outputImages.isEmpty()) { + return relsXml; + } + StringBuilder additions = new StringBuilder(); + for (Map.Entry e : outputImages.entrySet()) { + String id = e.getKey(); + ImageOutPart part = e.getValue(); + additions.append(""); + } + // inject before + int close = relsXml.lastIndexOf(""); + if (close < 0) { + // malformed - return as-is + return relsXml; + } + return relsXml.substring(0, close) + additions.toString() + relsXml.substring(close); + } + + // ===== image link resolution (getImage?id=...) — reused verbatim ===== + + private String replacement(String content) { + SimpleDateFormat formatter = DATE_FORMAT.get(); + + String assessors_nl = this.assessorsLines; + String assessors_comma = this.assessorsComma; + String assessors_bullets = this.assessorsBullets; + + content = content.replaceAll("\\$\\{asmtName\\}", + this.assessment.getName() == null ? "" : this.assessment.getName()); + content = content.replaceAll("\\$\\{asmtId\\}", + this.assessment.getId() == null ? "" : "" + this.assessment.getId()); + content = content.replaceAll("\\$\\{asmtAppId\\}", "" + this.assessment.getAppId()); + content = content.replaceAll("\\$\\{asmtAssessor\\}", this.assessment.getAssessor() == null ? "" + : (this.assessment.getAssessor().get(0).getFname() + " " + + this.assessment.getAssessor().get(0).getLname())); + content = content.replaceAll("\\$\\{asmtAssessor_Email\\}", + this.assessment.getAssessor() == null ? "" : (this.assessment.getAssessor().get(0).getEmail())); + content = content.replaceAll("\\$\\{asmtAssessors_Lines\\}", + this.assessment.getAssessor() == null ? "" : assessors_nl); + content = content.replaceAll("\\$\\{asmtAssessors_Comma\\}", + this.assessment.getAssessor() == null ? "" : assessors_comma); + content = content.replaceAll("\\$\\{asmtAssessors_Bullets\\}", + this.assessment.getAssessor() == null ? "" : assessors_bullets); + content = content.replaceAll("\\$\\{remediation\\}", this.assessment.getRemediation() == null ? "" + : (this.assessment.getRemediation().getFname() + " " + this.assessment.getRemediation().getLname())); + + content = content.replaceAll("\\$\\{asmtTeam\\}", + this.assessment.getAssessor() == null ? "" + : this.assessment.getAssessor().get(0).getTeam() == null ? "" + : this.assessment.getAssessor().get(0).getTeam().getTeamName().trim()); + content = content.replaceAll("\\$\\{asmtType\\}", + this.assessment.getType() == null ? "" : this.assessment.getType().getType().trim()); + content = replaceDateVariable(content, "today", new Date()); + content = content.replaceAll("\\$\\{asmtStandND\\}", formatter.format(this.assessment.getEnd())); + content = content.replaceAll("\\$\\{asmtAccessKey\\}", this.assessment.getGuid()); + content = content.replaceAll("\\$\\{totalOpenVulns\\}", this.totalOpenVulns); + content = content.replaceAll("\\$\\{totalClosedVulns\\}", this.totalClosedVulns); + + if (this.reportExtension.isExtended()) { + content = this.reportExtension.updateReport(this.assessment, content); + } + + content = loopReplace(content); + + content = FSUtils.jtidy(content); + if (maxImageWidth > 0 && content.contains("data:image/")) { + content = downscaleInlineImages(content); + } + content = this.replaceImageLinks(content); + return content; + } + + private String replaceImageLinks(String text) { + if (text.equals("")) { + return text; + } + if (!text.contains("getImage")) { + return text; + } + text = this.centerImages(text); + + String badImage = "]*src=[\"']getImage\\?id(=|=)undefined[\"'][^>]*>()?"; + text = text.replaceAll(badImage, ""); + + Pattern imagePattern = IMAGE_LINK_PATTERN; + Set referencedGuids = new HashSet<>(); + Matcher matcher = imagePattern.matcher(text); + while (matcher.find()) { + referencedGuids.add(matcher.group(2)); + } + if (referencedGuids.isEmpty()) { + return text; + } + + Map resolvedImages = resolveImages(referencedGuids); + + StringBuffer result = new StringBuffer(); + matcher = imagePattern.matcher(text); + while (matcher.find()) { + String guid = matcher.group(2); + String base64Image = resolvedImages.get(guid); + if (base64Image != null) { + String newImgTag = ""; + matcher.appendReplacement(result, Matcher.quoteReplacement(newImgTag)); + } else { + matcher.appendReplacement(result, ""); + } + } + matcher.appendTail(result); + return result.toString(); + } + + private Map resolveImages(Set referencedGuids) { + Map resolved = new HashMap<>(); + List toFetch = new ArrayList<>(); + for (String guid : referencedGuids) { + String cached = this.imageCache.get(guid); + if (cached != null) { + resolved.put(guid, cached); + } else if (!this.imagesNotFound.contains(guid)) { + toFetch.add(guid); + } + } + if (toFetch.isEmpty()) { + return resolved; + } + EntityManager em = HibHelper.getInstance().getEM(); + try { + for (String guid : toFetch) { + try { + Image img = (Image) em.createQuery("SELECT i FROM Image i WHERE i.guid = :guid") + .setParameter("guid", guid).getSingleResult(); + if (img != null && img.getBase64Image() != null) { + String base64 = ReportImageScaler.reportUri(img, maxImageWidth); + resolved.put(guid, base64); + cacheImage(guid, base64); + } else { + this.imagesNotFound.add(guid); + } + } catch (Exception e) { + this.imagesNotFound.add(guid); + } + } + } finally { + em.close(); + } + return resolved; + } + + private void cacheImage(String guid, String base64) { + if (base64.length() > MAX_IMAGE_CACHE_CHARS / 4) { + return; + } + this.imageCache.put(guid, base64); + this.imageCacheChars += base64.length(); + java.util.Iterator> eldest = this.imageCache.entrySet().iterator(); + while (this.imageCacheChars > MAX_IMAGE_CACHE_CHARS && this.imageCache.size() > 1 && eldest.hasNext()) { + Map.Entry entry = eldest.next(); + this.imageCacheChars -= entry.getValue().length(); + eldest.remove(); + } + } + + private String downscaleInlineImages(String content) { + List uris = extractInlineDataUris(content); + if (uris.isEmpty()) { + return content; + } + StringBuilder sb = new StringBuilder(content); + for (String uri : uris) { + String downscaled = this.inlineImagesDownscaled.remove(ReportImageScaler.hash(uri)); + if (downscaled == null) { + downscaled = ReportImageScaler.downscaleDataUri(uri, maxImageWidth); + } + if (!downscaled.equals(uri)) { + int idx = sb.indexOf(uri); + if (idx >= 0) { + sb.replace(idx, idx + uri.length(), downscaled); + } + } + } + return sb.toString(); + } + + private static List extractInlineDataUris(String content) { + List uris = new ArrayList<>(); + int idx = 0; + while ((idx = content.indexOf("data:image/", idx)) != -1) { + if (idx == 0) { + idx += 1; + continue; + } + char quote = content.charAt(idx - 1); + if (quote != '"' && quote != '\'') { + idx += 1; + continue; + } + int end = content.indexOf(quote, idx); + if (end == -1) { + break; + } + uris.add(content.substring(idx, end)); + idx = end; + } + return uris; + } + + @ProfileMethod("DocxUtils2: parallel image fetch and downscale") + private void warmImageCache(List fields) { + MethodProfiler.ProfileContext context = MethodProfiler.start("DocxUtils2", "warmImageCache"); + try { + Set guids = new HashSet<>(); + Map inlineByHash = new HashMap<>(); + for (String[] field : fields) { + String content = field[2]; + if (content.contains("getImage")) { + Matcher m = IMAGE_LINK_PATTERN.matcher(content); + while (m.find()) { + String guid = m.group(2); + if (!this.imageCache.containsKey(guid) && !this.imagesNotFound.contains(guid)) { + guids.add(guid); + } + } + } + if (maxImageWidth > 0 && content.contains("data:image/")) { + for (String uri : extractInlineDataUris(content)) { + inlineByHash.put(ReportImageScaler.hash(uri), uri); + } + } + } + + final Map rawImages = new HashMap<>(); + if (!guids.isEmpty()) { + EntityManager em = HibHelper.getInstance().getEM(); + try { + for (String guid : guids) { + try { + Image img = (Image) em.createQuery("SELECT i FROM Image i WHERE i.guid = :guid") + .setParameter("guid", guid).getSingleResult(); + if (img != null && img.getBase64Image() != null) { + if (ReportImageScaler.isReportReady(img, maxImageWidth)) { + // rendition prepared at upload — no decode work left + cacheImage(guid, ReportImageScaler.reportUri(img, maxImageWidth)); + } else { + rawImages.put(guid, img.getBase64Image()); + } + } else { + this.imagesNotFound.add(guid); + } + } catch (Exception e) { + this.imagesNotFound.add(guid); + } + } + } finally { + em.close(); + } + } + if (rawImages.isEmpty() && inlineByHash.isEmpty()) { + return; + } + + int threads = Math.min(8, Math.max(2, Runtime.getRuntime().availableProcessors() - 1)); + ExecutorService pool = Executors.newFixedThreadPool(threads); + final ConcurrentHashMap scaledGuids = new ConcurrentHashMap<>(); + try { + for (final Map.Entry entry : rawImages.entrySet()) { + pool.submit(new Runnable() { + @Override + public void run() { + scaledGuids.put(entry.getKey(), + ReportImageScaler.downscaleDataUri(entry.getValue(), maxImageWidth)); + } + }); + } + for (final Map.Entry entry : inlineByHash.entrySet()) { + pool.submit(new Runnable() { + @Override + public void run() { + String downscaled = ReportImageScaler.downscaleDataUri(entry.getValue(), maxImageWidth); + if (!downscaled.equals(entry.getValue())) { + inlineImagesDownscaled.put(entry.getKey(), downscaled); + } + } + }); + } + } finally { + pool.shutdown(); + try { + pool.awaitTermination(30, TimeUnit.MINUTES); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + for (Map.Entry entry : scaledGuids.entrySet()) { + cacheImage(entry.getKey(), entry.getValue()); + } + } finally { + context.end(); + } + } + + private String centerImages(String content) { + content = content.replaceAll("(]*?)\\s*/?>(?!)", "$1>"); + content = content.replaceAll("


", ""); + return content; + } + + public static String replaceDateVariable(String text, String key, Date date) { + String patternStr = "\\$\\{\\s*" + Pattern.quote(key) + "(?:\\s+([^}]+))?\\s*\\}"; + Pattern pattern = Pattern.compile(patternStr); + Matcher matcher = pattern.matcher(text); + StringBuffer result = new StringBuffer(); + while (matcher.find()) { + String dateFormat = matcher.group(1); + if (dateFormat == null || dateFormat.trim().isEmpty()) { + dateFormat = "MM/dd/yyyy"; + } else { + dateFormat = dateFormat.trim(); + } + try { + SimpleDateFormat formatter = new SimpleDateFormat(dateFormat); + String formattedDate = formatter.format(date); + matcher.appendReplacement(result, formattedDate); + } catch (Exception e) { + matcher.appendReplacement(result, matcher.group(0)); + } + } + matcher.appendTail(result); + return result.toString(); + } + + public String replaceFigureVariables(String text, int index) { + Pattern pattern = Pattern.compile("\\$\\{Figure#\\.(\\d+)\\}"); + Matcher matcher = pattern.matcher(text); + StringBuffer result = new StringBuffer(); + while (matcher.find()) { + String subNumber = matcher.group(1); + String replacement = "Figure " + index + "." + subNumber; + matcher.appendReplacement(result, replacement); + } + matcher.appendTail(result); + return result.toString(); + } + + // ===== per-vuln XML replacement - same string-based logic as DocxUtils ===== + + private static String replaceXml(String xml, Vulnerability v, + HashMap customFieldMap, HashMap colorMap, + HashMap cellMap, HashMap fillMap, + int count, int sevIndex) { + SimpleDateFormat formatter = DATE_FORMAT.get(); + String nxml = replaceAll(xml, "vulnName", v.getName(), true); + nxml = replaceAll(nxml, "severity", v.getOverallStr(), true); + nxml = replaceAll(nxml, "impact", v.getImpactStr(), true); + nxml = replaceAll(nxml, "likelihood", v.getLikelyhoodStr(), true); + nxml = replaceAll(nxml, "cvssScore", v.getCvssScore(), true); + nxml = replaceAll(nxml, "cvssString", v.getCvssString(), true); + nxml = replaceAll(nxml, "tracking", v.getTracking(), true); + if (v.getOpened() != null) { + nxml = replaceAll(nxml, "openedAt", formatter.format(v.getOpened()), false); + } else { + nxml = replaceAll(nxml, "openedAt", "", false); + } + if (v.getClosed() != null) { + nxml = replaceAll(nxml, "closedAt", formatter.format(v.getClosed()), false); + } else { + nxml = replaceAll(nxml, "closedAt", "", false); + } + if (v.getDevClosed() != null) { + nxml = replaceAll(nxml, "closedInDevAt", formatter.format(v.getDevClosed()), false); + } else { + nxml = replaceAll(nxml, "closedInDevAt", "", false); + } + try { + nxml = replaceAll(nxml, "vid", "" + v.getId(), false); + } catch (Exception ex) { + } + nxml = replaceAll(nxml, "category", + v.getCategory() == null ? "UnCategorized" : v.getCategory().getName(), true); + if (v.getClosed() == null) { + nxml = replaceAll(nxml, "remediationStatus", "Open", false); + } else { + nxml = replaceAll(nxml, "remediationStatus", "Closed", false); + } + nxml = replaceAll(nxml, "count", "" + count, false); + nxml = replaceAll(nxml, "loop", "", false); + nxml = nxml.replaceAll("\\$\\{loop\\-[0-9]+\\}", ""); + + if (v.getOverallStr() != null && !v.getOverallStr().equals("")) { + nxml = replaceAll(nxml, "sevId", "" + v.getOverallStr().charAt(0) + "V" + sevIndex, false); + } else { + nxml = replaceAll(nxml, "sevId", "V" + sevIndex, false); + } + + if (v.getCustomFields() != null) { + for (CustomField cf : v.getCustomFields()) { + if (cf.getType().getFieldType() < 3) { + nxml = StringUtils.replace(nxml, "${cf" + cf.getType().getVariable() + "}", CData(cf.getValue())); + if (customFieldMap.containsKey(cf.getType().getVariable()) + && colorMap.containsKey(cf.getValue())) { + String colorMatch = customFieldMap.get(cf.getType().getVariable()); + String color = colorMap.get(cf.getValue()); + if (colorMatch != null && colorMatch != "" && color != null && color != "") { + nxml = StringUtils.replace(nxml, "w:val=\"" + colorMatch + "\"", "w:val=\"" + color + "\""); + } + } + if (customFieldMap.containsKey(cf.getType().getVariable()) + && cellMap.containsKey(cf.getValue())) { + String colorMatch = customFieldMap.get(cf.getType().getVariable()); + String color = cellMap.get(cf.getValue()); + if (colorMatch != null && colorMatch != "" && color != null && color != "") { + nxml = StringUtils.replace(nxml, "w:fill=\"" + colorMatch + "\"", "w:fill=\"" + color + "\""); + } + } + } + } + } + + nxml = StringUtils.replace(nxml, "w:color=\"FAC701\"", "w:color=\"" + colorMap.get(v.getOverallStr()) + "\""); + nxml = StringUtils.replace(nxml, "w:color=\"FAC701\"", "w:color=\"" + colorMap.get(v.getOverallStr()) + "\""); + nxml = StringUtils.replace(nxml, "w:color=\"FAC702\"", "w:color=\"" + colorMap.get(v.getLikelyhoodStr()) + "\""); + nxml = StringUtils.replace(nxml, "w:color=\"FAC703\"", "w:color=\"" + colorMap.get(v.getImpactStr()) + "\""); + nxml = StringUtils.replace(nxml, "w:fill=\"FAC701\"", "w:fill=\"" + cellMap.get(v.getOverallStr()) + "\""); + nxml = StringUtils.replace(nxml, "w:fill=\"FAC702\"", "w:fill=\"" + cellMap.get(v.getLikelyhoodStr()) + "\""); + nxml = StringUtils.replace(nxml, "w:fill=\"FAC703\"", "w:fill=\"" + cellMap.get(v.getImpactStr()) + "\""); + nxml = StringUtils.replace(nxml, "w:val=\"FAC701\"", "w:val=\"" + colorMap.get(v.getOverallStr()) + "\""); + nxml = StringUtils.replace(nxml, "w:val=\"FAC702\"", "w:val=\"" + colorMap.get(v.getLikelyhoodStr()) + "\""); + nxml = StringUtils.replace(nxml, "w:val=\"FAC703\"", "w:val=\"" + colorMap.get(v.getImpactStr()) + "\""); + return nxml; + } + + private static String replaceAll(String original, String pattern, String replacement, Boolean wrapCDATA) { + if (wrapCDATA) { + return StringUtils.replace(original, "${" + pattern + "}", CData(replacement)); + } else { + return StringUtils.replace(original, "${" + pattern + "}", replacement); + } + } + + private static String replaceAllCf(String original, Vulnerability v) { + if (v.getCustomFields() != null) { + for (CustomField cf : v.getCustomFields()) { + try { + original = replaceAll(original, "cf" + cf.getType().getVariable(), cf.getValue(), true); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + } + return original; + } + + private static String getRecommendation(Vulnerability v) { + String rec = ""; + if (v.getRecommendation() == null && v.getDefaultVuln() != null) { + rec = v.getDefaultVuln().getRecommendation(); + } else if (v.getRecommendation() != null) { + rec = v.getRecommendation(); + } + if (!rec.isEmpty()) { + return replaceAllCf(rec, v); + } + return rec; + } + + private static String getDescription(Vulnerability v) { + String desc = ""; + if (v.getDescription() == null && v.getDefaultVuln() != null) { + desc = v.getDefaultVuln().getDescription(); + } else if (v.getDescription() != null) { + desc = v.getDescription(); + } + if (!desc.isEmpty()) { + return replaceAllCf(desc, v); + } + return desc; + } + + private static String getDetails(Vulnerability v) { + String details = ""; + if (v.getDetails() != null) { + return v.getDetails(); + } + if (!details.isEmpty()) { + return replaceAllCf(details, v); + } + return details; + } + + // ===== stubs - features not yet ported ===== + + /** + * Hyperlink rewriting is not yet ported. Templates that use + * ${cf link} placeholders for clickable URLs will surface them + * as plain text until this is implemented. + */ + private String replaceHyperlinks(String xml, Map map) { + return xml; + } + + /** + * ${pageBreak} placeholders and ${TOC} generation are not yet ported. + * The caller can still produce a ToC update via LibreOffice on the + * final docx - which is the same approach the production flow uses when + * finalize() is invoked. + */ + private String removePageBreaks(String xml) { + return xml; + } +} diff --git a/src/com/fuse/reporting/GenerateReport.java b/src/com/fuse/reporting/GenerateReport.java index 5800bd17..a0ae4be8 100644 --- a/src/com/fuse/reporting/GenerateReport.java +++ b/src/com/fuse/reporting/GenerateReport.java @@ -166,6 +166,14 @@ public String generateRetestDocxReport(Long id, EntityManager em, String host) { public String [] generateDocxReport(Long id, EntityManager em, Boolean isRetest) { + return this.generateDocxReport(id, em, isRetest, true); + } + + // finalize=false skips ReportFeatures.finalizeReport and returns the raw + // docx, so callers can run the ToC update and any format conversions in a + // single LibreOffice pass instead of one per output format + public String [] generateDocxReport(Long id, EntityManager em, Boolean isRetest, Boolean finalize) { + ReportOptions RPO = FSUtils.getOrCreateReportOptionsIfNotExist(em); String customCSS = css + (RPO == null ? "" : RPO.getBodyCss()); @@ -205,11 +213,10 @@ public String generateRetestDocxReport(Long id, EntityManager em, String host) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); mlp.save(baos); byte[] finalReport = baos.toByteArray(); - - String docx = Base64 - .encodeBase64String( - ReportFeatures.finalizeReport(finalReport, "docx") - ); + if (finalize) { + finalReport = ReportFeatures.finalizeReport(finalReport, "docx"); + } + String docx = Base64.encodeBase64String(finalReport); return new String [] {docx, "docx"}; } catch (Exception ex) { @@ -218,6 +225,67 @@ public String generateRetestDocxReport(Long id, EntityManager em, String host) { return null; } + // experimental raw-XML path: DocxUtils2 mutates document.xml as a string, + // bypassing the per-vuln JAXB marshal/unmarshal and growing-tree traversal + // of DocxUtils. HTML fragments (desc/rec/details) still go through the + // XHTML importer against a scratch package; image rIds are remapped into + // the output zip. Hyperlinks and page-break tags are not yet ported. + // + // returns the same {base64, "docx"} shape as generateDocxReport so report + // endpoints can swap implementations behind a flag. + public String [] generateDocxReport2(Long id, EntityManager em, Boolean isRetest, Boolean finalize) { + + ReportOptions RPO = FSUtils.getOrCreateReportOptionsIfNotExist(em); + String customCSS = css + (RPO == null ? "" : RPO.getBodyCss()); + + try { + Assessment a = (Assessment) em.createQuery("from Assessment where id = :id").setParameter("id", id) + .getResultList().stream().findFirst().orElse(null); + for (Vulnerability v : a.getVulns()) { + v.updateRiskLevels(em); + } + String mongoQuery = "{ 'type_id' : " + a.getType().getId() + ", 'team_id' : " + + a.getAssessor().get(0).getTeam().getId() + ", 'retest' : " + isRetest + " }"; + ReportTemplates base = (ReportTemplates) em.createNativeQuery(mongoQuery, ReportTemplates.class) + .getSingleResult(); + + InputStream is = null; + if (!base.getSaveInDB()) { + ReportTemplate report = (new ReportTemplateFactory()).getReportTemplate(); + is = report.getTemplate(base.getFilename()); + } else { + is = base.getTemplate(); + } + + // DocxUtils2 reads the template InputStream itself; the scratch + // package is loaded inside generateReport after VariablePrepare + DocxUtils2 gen = new DocxUtils2(em.getEntityManagerFactory(), a); + gen.FONT = RPO.getFont(); + MethodProfiler.setEnabled(true); + byte[] finalReport = gen.generateReport(is, customCSS); + MethodProfiler.printReport(); + MethodProfiler.clearStats(); + + // ToC generation still happens via docx4j - re-load the produced + // docx, install the TOC field at the ${TOC} placeholder, save + // back. This is O(1) with vuln count so it's not a hot path. + WordprocessingMLPackage mlp = WordprocessingMLPackage.load(new java.io.ByteArrayInputStream(finalReport)); + new DocxUtils(mlp, a).tocGenerator(mlp); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + mlp.save(baos); + finalReport = baos.toByteArray(); + + if (finalize) { + finalReport = ReportFeatures.finalizeReport(finalReport, "docx"); + } + String docx = Base64.encodeBase64String(finalReport); + return new String[] { docx, "docx" }; + } catch (Exception ex) { + ex.printStackTrace(); + } + return null; + } + public static Assessment createTestAssessment(Teams t, AssessmentType type, List riskLevels, String[] sections) { int index = 1; diff --git a/src/com/fuse/utils/ReportImageScaler.java b/src/com/fuse/utils/ReportImageScaler.java new file mode 100644 index 00000000..fc2ee099 --- /dev/null +++ b/src/com/fuse/utils/ReportImageScaler.java @@ -0,0 +1,225 @@ +package com.fuse.utils; + +import java.awt.Graphics2D; +import java.awt.RenderingHints; +import java.awt.Transparency; +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Base64; + +import javax.imageio.ImageIO; + +import com.fuse.dao.Image; + +/** + * Downscales oversized report images before they are embedded into + * documents. Full-resolution screenshots are the dominant cost of report + * generation — XHTML conversion time, docx size, heap and LibreOffice/PDF + * time all scale with pixel count — while the rendered image is page width + * or narrower anyway. + * + * The maximum width is configurable with the FACTION_REPORT_IMAGE_MAX_WIDTH + * environment variable (or system property). A value of 0 or less disables + * downscaling entirely. + */ +public class ReportImageScaler { + + public static final int DEFAULT_MAX_WIDTH = 1600; + + // images larger than this many bytes are re-encoded even when their + // width already fits the cap + private static final int REENCODE_BYTE_THRESHOLD = 512 * 1024; + + public static int configuredMaxWidth() { + String conf = System.getProperty("FACTION_REPORT_IMAGE_MAX_WIDTH"); + if (conf == null || conf.trim().isEmpty()) { + conf = System.getenv("FACTION_REPORT_IMAGE_MAX_WIDTH"); + } + if (conf == null || conf.trim().isEmpty()) { + return DEFAULT_MAX_WIDTH; + } + try { + return Integer.parseInt(conf.trim()); + } catch (NumberFormatException e) { + System.err.println("Invalid FACTION_REPORT_IMAGE_MAX_WIDTH '" + conf + "', using " + + DEFAULT_MAX_WIDTH); + return DEFAULT_MAX_WIDTH; + } + } + + /** + * True when the stored rendition was prepared for this width cap and + * can be embedded without any decode work. + */ + public static boolean isReportReady(Image img, int maxWidth) { + return img != null && img.getReportWidth() != null && img.getReportWidth().intValue() == maxWidth; + } + + /** + * Computes and stores the report-ready rendition on the image entity; + * the caller persists. Returns true when the entity was modified. A + * null rendition with a recorded width means "checked — the original + * is already report-ready", so the original isn't duplicated. + */ + public static boolean prepareReportRendition(Image img) { + int maxWidth = configuredMaxWidth(); + if (img == null || maxWidth <= 0 || img.getBase64Image() == null) { + return false; + } + if (isReportReady(img, maxWidth)) { + return false; + } + String scaled = downscaleDataUri(img.getBase64Image(), maxWidth); + img.setReportImage(scaled.equals(img.getBase64Image()) ? null : scaled); + img.setReportWidth(maxWidth); + return true; + } + + /** + * The data URI report generation should embed for this image: the + * stored rendition when it matches the current width cap, otherwise a + * live downscale of the original (covers images uploaded before the + * backfill has reached them and width-cap changes). + */ + public static String reportUri(Image img, int maxWidth) { + if (img == null) { + return null; + } + if (maxWidth <= 0) { + return img.getBase64Image(); + } + if (isReportReady(img, maxWidth)) { + return img.getReportImage() != null ? img.getReportImage() : img.getBase64Image(); + } + return downscaleDataUri(img.getBase64Image(), maxWidth); + } + + /** + * Data URI in, data URI out. Returns the input unchanged when the image + * is already narrow enough, is not a re-encodable format (gif/webp/svg), + * or anything at all goes wrong — a full-size image is always preferable + * to a broken report. + */ + public static String downscaleDataUri(String dataUri, int maxWidth) { + if (maxWidth <= 0 || dataUri == null) { + return dataUri; + } + try { + if (!dataUri.startsWith("data:image/")) { + return dataUri; + } + int comma = dataUri.indexOf(','); + if (comma < 0) { + return dataUri; + } + String header = dataUri.substring(0, comma); + String format; + if (header.contains("image/png")) { + format = "png"; + } else if (header.contains("image/jpeg") || header.contains("image/jpg")) { + format = "jpg"; + } else { + return dataUri; + } + byte[] bytes = Base64.getMimeDecoder().decode(dataUri.substring(comma + 1)); + BufferedImage img = ImageIO.read(new ByteArrayInputStream(bytes)); + if (img == null) { + return dataUri; + } + boolean needsResize = img.getWidth() > maxWidth; + // heavy-but-narrow images (large screenshots that fit the width + // cap) still bloat the document; re-encode them at original size + boolean heavy = bytes.length > REENCODE_BYTE_THRESHOLD; + if (!needsResize && !heavy) { + return dataUri; + } + int w = needsResize ? maxWidth : img.getWidth(); + int h = needsResize + ? Math.max(1, (int) Math.round((double) img.getHeight() * maxWidth / img.getWidth())) + : img.getHeight(); + boolean opaque = img.getTransparency() == Transparency.OPAQUE || "jpg".equals(format); + BufferedImage scaled = new BufferedImage(w, h, + opaque ? BufferedImage.TYPE_INT_RGB : BufferedImage.TYPE_INT_ARGB); + Graphics2D g = scaled.createGraphics(); + g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR); + g.setRenderingHint(RenderingHints.KEY_RENDERING, RenderingHints.VALUE_RENDER_QUALITY); + g.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON); + g.drawImage(img, 0, 0, w, h, null); + g.dispose(); + + // keep whichever encoding is smallest: downscaled PNG keeps + // crisp text, JPEG wins by a wide margin on dense content that + // PNG cannot compress once scaling has smoothed it + byte[] best = null; + String bestMime = null; + ByteArrayOutputStream png = new ByteArrayOutputStream(); + if (ImageIO.write(scaled, "png", png) && png.size() < bytes.length) { + best = png.toByteArray(); + bestMime = "image/png"; + } + if (opaque) { + byte[] jpg = encodeJpeg(scaled, 0.85f); + if (jpg != null && jpg.length < bytes.length && (best == null || jpg.length < best.length)) { + best = jpg; + bestMime = "image/jpeg"; + } + } + if (best == null) { + // nothing beat the original bytes; keep it + return dataUri; + } + if (!needsResize && best.length > bytes.length * 0.8) { + // a pure re-encode must earn its (possibly lossy) keep + return dataUri; + } + return "data:" + bestMime + ";base64," + Base64.getEncoder().encodeToString(best); + } catch (Throwable t) { + // never fail a report over an image + return dataUri; + } + } + + private static byte[] encodeJpeg(BufferedImage img, float quality) { + javax.imageio.ImageWriter writer = null; + try { + writer = ImageIO.getImageWritersByFormatName("jpg").next(); + javax.imageio.ImageWriteParam param = writer.getDefaultWriteParam(); + param.setCompressionMode(javax.imageio.ImageWriteParam.MODE_EXPLICIT); + param.setCompressionQuality(quality); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + javax.imageio.stream.ImageOutputStream ios = ImageIO.createImageOutputStream(out); + writer.setOutput(ios); + writer.write(null, new javax.imageio.IIOImage(img, null, null), param); + ios.close(); + return out.toByteArray(); + } catch (Throwable t) { + return null; + } finally { + if (writer != null) { + writer.dispose(); + } + } + } + + /** Cheap stable key for caching downscale results of large URI strings. */ + public static String hash(String value) { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] digest = md.digest(value.getBytes(StandardCharsets.UTF_8)); + StringBuilder hex = new StringBuilder(); + for (byte b : digest) { + String h = Integer.toHexString(0xff & b); + if (h.length() == 1) { + hex.append('0'); + } + hex.append(h); + } + return hex.toString(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/test/org/fuse/docx/DocxAssessmentVarsParityTest.java b/test/org/fuse/docx/DocxAssessmentVarsParityTest.java new file mode 100644 index 00000000..cc158943 --- /dev/null +++ b/test/org/fuse/docx/DocxAssessmentVarsParityTest.java @@ -0,0 +1,171 @@ +package org.fuse.docx; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import javax.persistence.EntityManager; +import javax.persistence.EntityManagerFactory; +import javax.persistence.Query; + +import org.docx4j.openpackaging.packages.WordprocessingMLPackage; +import org.junit.Test; +import org.mockito.Mockito; + +import com.fuse.dao.AppStore; +import com.fuse.dao.Assessment; +import com.fuse.dao.AssessmentType; +import com.fuse.dao.CustomField; +import com.fuse.dao.CustomType; +import com.fuse.dao.RiskLevel; +import com.fuse.dao.Teams; +import com.fuse.dao.Vulnerability; +import com.fuse.reporting.DocxPrecompiler; +import com.fuse.reporting.DocxUtils; +import com.fuse.reporting.GenerateReport; + +/** + * Guards the ordering change that runs replaceAssessment BEFORE the + * findings are inserted (so its whole-document marshal covers only the + * template): every assessment-level variable that used to be resolved by + * the document-wide pass must now be resolved inside finding content by + * the per-field paths — replacement() for live conversion and + * applyAssessmentVarsToXml for the pre-compiled cache. A variable that + * leaks through either path shows up here as a literal ${...} in the + * output. + */ +public class DocxAssessmentVarsParityTest { + + @Test + public void assessmentVariablesResolveInsideFindings() throws Exception { + Teams team = new Teams(); + team.setId(123l); + team.setTeamName("Hacking Team"); + AssessmentType type = new AssessmentType(); + type.setId(1235l); + type.setType("Assessment Type"); + + List levels = new ArrayList<>(); + String[] risk = { "Informational", "Recommended", "Low", "Medium", "High", "Critical" }; + for (int i = 0; i < 10; i++) { + RiskLevel level = new RiskLevel(); + level.setRiskId(i); + if (i < risk.length) + level.setRisk(risk[i]); + levels.add(level); + } + + Assessment assessment = GenerateReport.createTestAssessment(team, type, levels, new String[] { "S1" }); + assessment.setGuid("test-guid"); + Date start = new java.util.GregorianCalendar(2026, 0, 5).getTime(); + Date end = new java.util.GregorianCalendar(2026, 0, 23).getTime(); + assessment.setStart(start); + assessment.setEnd(end); + + // assessment-level TEXT custom field (fieldType < 3) + CustomType engagementType = new CustomType(); + engagementType.setVariable("engagement"); + engagementType.setFieldType(1); + CustomField engagement = new CustomField(); + engagement.setType(engagementType); + engagement.setValue("ENG-42"); + List asmtCfs = new ArrayList<>(); + asmtCfs.add(engagement); + assessment.setCustomFields(asmtCfs); + + // every assessment-scoped token the doc-wide pass used to resolve + // inside findings, in both cached and live fields + String varSoup = "name=${asmtName} start=${asmtStart} end=${asmtEnd} eng=${cfengagement}" + + " team=${asmtTeam} type=${asmtType} open=${totalOpenVulns}"; + assessment.getVulns().clear(); + for (int i = 1; i <= 4; i++) { + Vulnerability v = new Vulnerability(); + v.setLevels(levels); + v.setId((long) i); + v.setName("VarIssue " + i); + v.setImpact(3l); + v.setLikelyhood(3l); + v.setOverall(3l); + v.setCvssScore("8.3"); + v.setCvssString("CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:N/VC:H/VI:L/VA:L/SC:N/SI:N/SA:N"); + v.setTracking("VID-" + i); + v.setAssessmentId(1l); + v.setCustomFields(new ArrayList()); + v.setDescription("Desc" + i + " " + varSoup); + v.setRecommendation("Rec" + i + "
  • fix ${asmtName}
"); + v.setDetails("Details" + i + " " + varSoup); + assessment.getVulns().add(v); + } + + // half cached, half live — both substitution paths must resolve + DocxPrecompiler pre = new DocxPrecompiler("Calibri", "p { margin: 0; }"); + pre.compile(assessment.getVulns().get(0)); + pre.compile(assessment.getVulns().get(1)); + + WordprocessingMLPackage mlp = WordprocessingMLPackage.createPackage(); + mlp.getMainDocumentPart().addParagraphOfText("Template header: ${asmtName} ${asmtStart} ${cfengagement}"); + mlp.getMainDocumentPart().addParagraphOfText("${fiBegin}"); + mlp.getMainDocumentPart().addParagraphOfText("${vulnName}"); + mlp.getMainDocumentPart().addParagraphOfText("${desc}"); + mlp.getMainDocumentPart().addParagraphOfText("${rec}"); + mlp.getMainDocumentPart().addParagraphOfText("${details}"); + mlp.getMainDocumentPart().addParagraphOfText("${fiEnd}"); + mlp.getMainDocumentPart().addParagraphOfText(""); + mlp.getMainDocumentPart().addParagraphOfText("End of report"); + + EntityManagerFactory emf = Mockito.mock(EntityManagerFactory.class); + EntityManager em = Mockito.mock(EntityManager.class); + Query query = Mockito.mock(Query.class); + Mockito.when(emf.createEntityManager()).thenReturn(em); + Mockito.when(em.createQuery("from AppStore order by order")).thenReturn(query); + Mockito.when(query.getResultList()).thenReturn(new ArrayList()); + + DocxUtils genDoc = new DocxUtils(emf, mlp, assessment); + genDoc.FONT = "Calibri"; + mlp = genDoc.generateDocx("p { margin: 0; }"); + + java.io.StringWriter sw = new java.io.StringWriter(); + org.docx4j.TextUtils.extractText(mlp.getMainDocumentPart().getContents(), sw); + String docText = sw.toString(); + + String startStr = new SimpleDateFormat("MM/dd/yyyy").format(start); + String endStr = new SimpleDateFormat("MM/dd/yyyy").format(end); + + for (int i = 1; i <= 4; i++) { + org.junit.Assert.assertTrue("missing vuln " + i, docText.contains("VarIssue " + i)); + org.junit.Assert.assertTrue("missing desc " + i, docText.contains("Desc" + i)); + } + // no unresolved assessment tokens anywhere — template or findings + org.junit.Assert.assertFalse("leaked ${asmt token: " + snippetAround(docText, "${asmt"), + docText.contains("${asmt")); + org.junit.Assert.assertFalse("leaked ${cfengagement token", + docText.contains("${cfengagement")); + org.junit.Assert.assertFalse("leaked ${totalOpenVulns token", + docText.contains("${totalOpenVulns")); + // resolved values present in finding content (name= prefix pins the + // occurrence to the varSoup inside desc/details, not the template) + org.junit.Assert.assertTrue("asmtName not resolved in findings", + docText.contains("name=Test PCI Assessment")); + org.junit.Assert.assertTrue("asmtStart not resolved in findings", + docText.contains("start=" + startStr)); + org.junit.Assert.assertTrue("asmtEnd not resolved in findings", + docText.contains("end=" + endStr)); + org.junit.Assert.assertTrue("assessment custom field not resolved in findings", + docText.contains("eng=ENG-42")); + org.junit.Assert.assertTrue("asmtTeam not resolved in findings", + docText.contains("team=Hacking Team")); + org.junit.Assert.assertTrue("totalOpenVulns not resolved in findings", + docText.contains("open=4")); + // template paragraph resolved by replaceAssessment as before + org.junit.Assert.assertTrue("template header not resolved", + docText.contains("Template header: Test PCI Assessment " + startStr + " ENG-42")); + } + + private static String snippetAround(String text, String needle) { + int idx = text.indexOf(needle); + if (idx < 0) + return ""; + return text.substring(Math.max(0, idx - 40), Math.min(text.length(), idx + 60)); + } +} diff --git a/test/org/fuse/docx/DocxListNumberingTest.java b/test/org/fuse/docx/DocxListNumberingTest.java new file mode 100644 index 00000000..294fcbec --- /dev/null +++ b/test/org/fuse/docx/DocxListNumberingTest.java @@ -0,0 +1,410 @@ +package org.fuse.docx; + +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.List; + +import javax.persistence.EntityManager; +import javax.persistence.EntityManagerFactory; +import javax.persistence.Query; + +import org.docx4j.openpackaging.packages.WordprocessingMLPackage; +import org.junit.Test; +import org.mockito.Mockito; + +import com.fuse.dao.AppStore; +import com.fuse.dao.Assessment; +import com.fuse.dao.AssessmentType; +import com.fuse.dao.CustomField; +import com.fuse.dao.RiskLevel; +import com.fuse.dao.Teams; +import com.fuse.dao.Vulnerability; +import com.fuse.reporting.DocxPrecompiler; +import com.fuse.reporting.DocxUtils; +import com.fuse.reporting.GenerateReport; + +/** + * Guards the bullet-list numbering behavior: bullets from different + * findings must never render as one continuing numbered sequence — not + * from live batched conversion, and not from the pre-compiled cache + * (whose v1 format stored bare scratch-package numIds that resolved + * against the report template's decimal numbering). + */ +public class DocxListNumberingTest { + + @Test + public void liveConversionKeepsBulletsBullets() throws Exception { + Assessment assessment = buildAssessment(); + WordprocessingMLPackage mlp = generate(assessment); + String docXml = org.docx4j.XmlUtils.marshaltoString( + mlp.getMainDocumentPart().getContents(), true, true); + int checked = verifyBullets(mlp, docXml); + System.out.println("live: verified bullet items: " + checked); + org.junit.Assert.assertTrue("expected many bullet items", checked >= 100); + } + + @Test + public void cachedFieldsKeepBulletsBullets() throws Exception { + Assessment assessment = buildAssessment(); + + // one vuln gets an inline screenshot so the cached-image path runs: + // precompile embeds it as a data URI, report time must create a real + // image part (probe-free) and a valid relationship + Vulnerability v3 = assessment.getVulns().get(2); + v3.setDetails(v3.getDetails() + ""); + + // pre-compile every vuln the way the save hooks do + DocxPrecompiler pre = new DocxPrecompiler("Calibri", "p { margin: 0; }"); + int compiled = 0; + for (Vulnerability v : assessment.getVulns()) { + if (pre.compile(v)) { + compiled++; + } + } + org.junit.Assert.assertTrue("precompiler should compile the fixture vulns", compiled >= 25); + + for (Vulnerability v : assessment.getVulns()) { + if (v.getId() == 1L) { + // prove the cache is actually served: tamper with the cached + // text (hash still matches the entity content) — the marker + // can only reach the document via the cached path + org.junit.Assert.assertNotNull(v.getCachedDetailsXml()); + v.setCachedDetailsXml(v.getCachedDetailsXml().replace("DTest1", "CACHEDMARK1")); + } + if (v.getId() == 2L) { + // v1-format poison: bare scratch-package numId with no + // carried definitions, hash forged to match. The legacy + // guard must refuse it and convert live. + v.setCachedDetailsXml("" + + "" + + "POISON"); + v.setCachedDetailsHash(DocxPrecompiler.contentHash("Calibri", + v.getDetails() != null ? v.getDetails() : "")); + } + } + + WordprocessingMLPackage mlp = generate(assessment); + String docXml = org.docx4j.XmlUtils.marshaltoString( + mlp.getMainDocumentPart().getContents(), true, true); + + org.junit.Assert.assertTrue("cached XML was not used (tamper marker missing)", + docXml.contains("CACHEDMARK1")); + java.io.StringWriter sw = new java.io.StringWriter(); + org.docx4j.TextUtils.extractText(mlp.getMainDocumentPart().getContents(), sw); + String docText = sw.toString(); + org.junit.Assert.assertTrue("colon text corrupted (':8080' lost)", + docText.contains("connect to host :8080 as user : admin")); + org.junit.Assert.assertFalse("namespace prefix leaked into text: 'w:8080'", + docText.contains("w:8080")); + org.junit.Assert.assertFalse("namespace prefix leaked into text: 'user w:'", + docText.contains("user w:")); + org.junit.Assert.assertFalse("legacy v1 cache format must not be used", + docXml.contains("POISON")); + org.junit.Assert.assertFalse("numbering tokens leaked into the document", + docXml.contains("FCT-NUM-")); + + int checked = verifyBullets(mlp, docXml); + System.out.println("cached: verified bullet items: " + checked); + org.junit.Assert.assertTrue("expected many bullet items", checked >= 100); + + // the cached image must have become a real, wired-up package part: + // a media entry, a relationship pointing at it, a registered content + // type, and an r:embed in the document referencing that relationship + ByteArrayOutputStream packageBytes = new ByteArrayOutputStream(); + mlp.save(packageBytes); + java.util.Set zipEntries = new java.util.TreeSet<>(); + java.util.zip.ZipInputStream zin = new java.util.zip.ZipInputStream( + new java.io.ByteArrayInputStream(packageBytes.toByteArray())); + String contentTypesXml = null; + String relsXml = null; + java.util.zip.ZipEntry ze; + while ((ze = zin.getNextEntry()) != null) { + zipEntries.add(ze.getName()); + if (ze.getName().equals("[Content_Types].xml") || ze.getName().equals("word/_rels/document.xml.rels")) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + int n; + while ((n = zin.read(buf)) > 0) { + out.write(buf, 0, n); + } + if (ze.getName().startsWith("[Content_Types]")) { + contentTypesXml = out.toString("UTF-8"); + } else { + relsXml = out.toString("UTF-8"); + } + } + } + String mediaEntry = null; + for (String name : zipEntries) { + if (name.startsWith("word/media/fctimage")) { + mediaEntry = name; + } + } + org.junit.Assert.assertNotNull("cached image part missing from package: " + zipEntries, mediaEntry); + org.junit.Assert.assertTrue("png content type not registered", + contentTypesXml != null && contentTypesXml.contains("image/png")); + org.junit.Assert.assertTrue("no relationship to the cached image part", + relsXml != null && relsXml.contains(mediaEntry.substring("word/".length()))); + org.junit.Assert.assertTrue("document does not embed the cached image", + docXml.contains("r:embed")); + } + + // small screenshot-like PNG as a data URI + private static String makePng(int w, int h) throws Exception { + java.awt.image.BufferedImage img = new java.awt.image.BufferedImage(w, h, + java.awt.image.BufferedImage.TYPE_INT_RGB); + java.awt.Graphics2D g = img.createGraphics(); + g.setColor(java.awt.Color.WHITE); + g.fillRect(0, 0, w, h); + g.setColor(java.awt.Color.DARK_GRAY); + g.drawString("evidence", 10, h / 2); + g.dispose(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + javax.imageio.ImageIO.write(img, "png", out); + return "data:image/png;base64," + java.util.Base64.getEncoder().encodeToString(out.toByteArray()); + } + + @Test + public void vulnTablePathKeepsBulletsBullets() throws Exception { + Assessment assessment = buildAssessment(); + // pre-compile only half the vulns so both the cache-hit branch and + // the batched-conversion branch of the table path run in one report + DocxPrecompiler pre = new DocxPrecompiler("Calibri", "p { margin: 0; }"); + for (Vulnerability v : assessment.getVulns()) { + if (v.getId() % 2 == 0) { + pre.compile(v); + } + } + + // production-shaped template: a ${vulnTable} marker row, a ${loop-1} + // row, and a second row carrying the HTML field placeholders + WordprocessingMLPackage mlp = WordprocessingMLPackage.createPackage(); + org.docx4j.wml.Tbl table = org.docx4j.model.table.TblFactory.createTable(3, 1, 4000); + setCellParagraphs(table, 0, "${vulnTable}"); + setCellParagraphs(table, 1, "${loop-1}", "${vulnName}"); + setCellParagraphs(table, 2, "${desc}", "${rec}", "${details}", "${cfpoc}"); + mlp.getMainDocumentPart().addObject(table); + mlp.getMainDocumentPart().addParagraphOfText("End of report"); + + // round-trip through bytes so JAXB parent pointers are set the way + // a real loaded template's are (indexOfRow walks parents) + ByteArrayOutputStream tpl = new ByteArrayOutputStream(); + mlp.save(tpl); + mlp = WordprocessingMLPackage.load(new java.io.ByteArrayInputStream(tpl.toByteArray())); + + EntityManagerFactory emf = Mockito.mock(EntityManagerFactory.class); + EntityManager em = Mockito.mock(EntityManager.class); + Query query = Mockito.mock(Query.class); + Mockito.when(emf.createEntityManager()).thenReturn(em); + Mockito.when(em.createQuery("from AppStore order by order")).thenReturn(query); + Mockito.when(query.getResultList()).thenReturn(new ArrayList()); + + DocxUtils genDoc = new DocxUtils(emf, mlp, assessment); + genDoc.FONT = "Calibri"; + mlp = genDoc.generateDocx("p { margin: 0; }"); + + java.io.StringWriter sw = new java.io.StringWriter(); + org.docx4j.TextUtils.extractText(mlp.getMainDocumentPart().getContents(), sw); + String docText = sw.toString(); + for (int i = 1; i <= 30; i++) { + org.junit.Assert.assertTrue("table missing vuln name " + i, docText.contains("Issue " + i)); + org.junit.Assert.assertTrue("table missing desc " + i, docText.contains("Desc" + i)); + org.junit.Assert.assertTrue("table missing rec " + i, docText.contains("Rec" + i)); + org.junit.Assert.assertTrue("table missing details " + i, docText.contains("bullet DTest" + i)); + } + org.junit.Assert.assertFalse("leaked field placeholder", docText.contains("${rec")); + org.junit.Assert.assertFalse("leaked field placeholder", docText.contains("${desc")); + org.junit.Assert.assertFalse("leaked field placeholder", docText.contains("${details")); + org.junit.Assert.assertFalse("leaked table marker", docText.contains("${vulnTable")); + org.junit.Assert.assertFalse("leaked loop marker", docText.contains("${loop")); + + String docXml = org.docx4j.XmlUtils.marshaltoString( + mlp.getMainDocumentPart().getContents(), true, true); + org.junit.Assert.assertFalse("numbering tokens leaked into the document", + docXml.contains("FCT-NUM-")); + int checked = verifyBullets(mlp, docXml); + System.out.println("vulnTable: verified bullet items: " + checked); + org.junit.Assert.assertTrue("expected many bullet items", checked >= 100); + } + + // replaces the single cell's content in the given table row with one + // paragraph per text value + private static void setCellParagraphs(org.docx4j.wml.Tbl table, int rowIndex, String... texts) { + org.docx4j.wml.Tr row = (org.docx4j.wml.Tr) table.getContent().get(rowIndex); + org.docx4j.wml.Tc cell = (org.docx4j.wml.Tc) org.docx4j.XmlUtils.unwrap(row.getContent().get(0)); + cell.getContent().clear(); + org.docx4j.wml.ObjectFactory factory = org.docx4j.jaxb.Context.getWmlObjectFactory(); + for (String text : texts) { + org.docx4j.wml.P p = factory.createP(); + org.docx4j.wml.R r = factory.createR(); + org.docx4j.wml.Text t = factory.createText(); + t.setValue(text); + r.getContent().add(t); + p.getContent().add(r); + cell.getContent().add(p); + } + } + + // ============================================================ + // Fixture + // ============================================================ + + private Assessment buildAssessment() { + Teams team = new Teams(); + team.setId(123l); + team.setTeamName("Hacking Team"); + AssessmentType type = new AssessmentType(); + type.setId(1235l); + type.setType("Assessment Type"); + + List levels = new ArrayList<>(); + String[] risk = { "Informational", "Recommended", "Low", "Medium", "High", "Critical" }; + for (int i = 0; i < 10; i++) { + RiskLevel level = new RiskLevel(); + level.setRiskId(i); + if (i < risk.length) + level.setRisk(risk[i]); + levels.add(level); + } + + Assessment assessment = GenerateReport.createTestAssessment(team, type, levels, new String[] { "S1" }); + assessment.setGuid("test-guid"); + assessment.getVulns().clear(); + // enough findings that convertFieldsBatched uses several chunks + // (25 fields per chunk), so cross-chunk numId collisions surface + for (int i = 1; i <= 30; i++) { + Vulnerability v = new Vulnerability(); + v.setLevels(levels); + v.setId((long) i); + v.setName("Issue " + i); + v.setImpact(3l); + v.setLikelyhood(3l); + v.setOverall(3l); + v.setCvssScore("8.3"); + v.setCvssString("CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:N/VC:H/VI:L/VA:L/SC:N/SI:N/SA:N"); + v.setTracking("VID-" + i); + v.setAssessmentId(1l); + v.setCustomFields(new ArrayList()); + // a type-3 (HTML) custom field with a bullet list — cached via + // cachedCfXml, so its numbering must survive the splice too + com.fuse.dao.CustomType pocType = new com.fuse.dao.CustomType(); + pocType.setVariable("poc"); + pocType.setFieldType(3); + CustomField poc = new CustomField(); + poc.setType(pocType); + poc.setValue("
  • bullet CF" + i + "-a
  • bullet CF" + i + "-b
"); + v.getCustomFields().add(poc); + // colon-adjacent text: the precompiler's namespace normalization + // must never touch text content (" :80" once became " w:80") + v.setDescription("Desc" + i + " connect to host :8080 as user : admin" + + "
  • bullet A" + i + "
  • bullet B" + i + "
"); + // production failure shapes: a numbered list followed by a bullet + // list in the same field, in both valid and editor-mangled + // (ul nested directly inside ol) markup + if (i % 2 == 0) { + v.setRecommendation("Rec" + i + + "
  1. step one

  2. step two

  3. step three

" + + "
  • bullet Test" + i + "
  • bullet Test2-" + i + "
"); + } else { + v.setRecommendation("Rec" + i + + "
  1. step one
  2. step two
  3. step three
  4. " + + "
    • bullet Test" + i + "
    • bullet Test2-" + i + "
"); + } + // SunEditor wraps every list item's content in

; the ol lives + // in the rec field, the ul in details — the production bug had + // the ul numbered as a continuation of the previous field's ol + v.setDetails("

  • bullet DTest" + i + "

  • bullet DTest2-" + i + + "


"); + assessment.getVulns().add(v); + } + return assessment; + } + + private WordprocessingMLPackage generate(Assessment assessment) throws Exception { + // use the real sample template when available — it ships a numbering + // part (decimal heading numbering on numIds 1-4) that a fresh + // package lacks, which is where allocation collisions surface + WordprocessingMLPackage mlp; + java.io.File template = new java.io.File(System.getProperty("user.dir") + "/src/test/sampletemplate.docx"); + if (template.exists()) { + mlp = WordprocessingMLPackage.load(template); + } else { + mlp = WordprocessingMLPackage.createPackage(); + } + mlp.getMainDocumentPart().addParagraphOfText("${fiBegin}"); + mlp.getMainDocumentPart().addParagraphOfText("${vulnName}"); + mlp.getMainDocumentPart().addParagraphOfText("${desc}"); + mlp.getMainDocumentPart().addParagraphOfText("${rec}"); + mlp.getMainDocumentPart().addParagraphOfText("${details}"); + mlp.getMainDocumentPart().addParagraphOfText("${cfpoc}"); + mlp.getMainDocumentPart().addParagraphOfText("${fiEnd}"); + mlp.getMainDocumentPart().addParagraphOfText(""); + mlp.getMainDocumentPart().addParagraphOfText("End of report"); + + EntityManagerFactory emf = Mockito.mock(EntityManagerFactory.class); + EntityManager em = Mockito.mock(EntityManager.class); + Query query = Mockito.mock(Query.class); + Mockito.when(emf.createEntityManager()).thenReturn(em); + Mockito.when(em.createQuery("from AppStore order by order")).thenReturn(query); + Mockito.when(query.getResultList()).thenReturn(new ArrayList()); + + DocxUtils genDoc = new DocxUtils(emf, mlp, assessment); + genDoc.FONT = "Calibri"; + mlp = genDoc.generateDocx("p { margin: 0; }"); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + mlp.save(baos); + java.nio.file.Files.write(java.nio.file.Paths.get("/tmp/list-numbering-test.docx"), baos.toByteArray()); + return mlp; + } + + // ============================================================ + // Verification: resolve every "bullet ..." list item through + // numId -> num -> abstractNum -> numFmt and require "bullet" + // ============================================================ + + private static int verifyBullets(WordprocessingMLPackage mlp, String docXml) throws Exception { + org.junit.Assert.assertNotNull("document has no numbering part", + mlp.getMainDocumentPart().getNumberingDefinitionsPart()); + String numberingXml = org.docx4j.XmlUtils.marshaltoString( + mlp.getMainDocumentPart().getNumberingDefinitionsPart().getContents(), true, true); + + java.util.Map abstractFmt = new java.util.HashMap<>(); + java.util.regex.Matcher am = java.util.regex.Pattern.compile( + "]*w:abstractNumId=\"(\\d+)\"[^>]*>(.*?)", + java.util.regex.Pattern.DOTALL).matcher(numberingXml); + while (am.find()) { + java.util.regex.Matcher f = java.util.regex.Pattern.compile("w:numFmt w:val=\"(\\w+)\"") + .matcher(am.group(2)); + abstractFmt.put(am.group(1), f.find() ? f.group(1) : "?"); + } + java.util.Map numToAbstract = new java.util.HashMap<>(); + java.util.regex.Matcher nm = java.util.regex.Pattern.compile( + "]*>\\s*", + java.util.regex.Pattern.DOTALL).matcher(docXml); + while (pm.find()) { + String p = pm.group(0); + StringBuilder text = new StringBuilder(); + java.util.regex.Matcher tm = java.util.regex.Pattern.compile("]*>([^<]*)").matcher(p); + while (tm.find()) { + text.append(tm.group(1)); + } + if (text.toString().contains("bullet ") || text.toString().contains("bullet A") + || text.toString().contains("bullet B") || text.toString().contains("bullet Test")) { + java.util.regex.Matcher nid = java.util.regex.Pattern.compile(" levels = new ArrayList<>(); + String[] risk = { "Informational", "Recommended", "Low", "Medium", "High", "Critical" }; + for (int i = 0; i < 10; i++) { + RiskLevel level = new RiskLevel(); + level.setRiskId(i); + if (i < risk.length) + level.setRisk(risk[i]); + levels.add(level); + } + + Assessment assessment = GenerateReport.createTestAssessment(team, type, levels, new String[] { "seed" }); + assessment.setGuid("perf-guid"); + assessment.getVulns().clear(); + + Random rnd = new Random(42); + for (int i = 1; i <= vulnCount; i++) { + Vulnerability v = new Vulnerability(); + v.setLevels(levels); + v.setId((long) i); + v.setName("Perf Issue " + i); + v.setImpact((long) (i % 5) + 1); + v.setLikelyhood((long) (i % 5) + 1); + v.setOverall((long) (i % 5) + 1); + v.setCvssScore("8.3"); + v.setCvssString("CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:N/VC:H/VI:L/VA:L/SC:N/SI:N/SA:N"); + v.setTracking("VID-" + i); + v.setAssessmentId(1l); + v.setCustomFields(new ArrayList()); + v.setDescription("Description for issue " + i + + " with a link and markup."); + v.setRecommendation("Recommendation for issue " + i + "
  • fix it
  • test it
"); + if (withImages) { + int imgW = Integer.parseInt(System.getProperty("faction.perf.imgw", "800")); + int imgH = Integer.parseInt(System.getProperty("faction.perf.imgh", "500")); + v.setDetails("Details for issue " + i + "
payload

" + + "
"); + } else { + v.setDetails("Details for issue " + i + "
payload

plain text details"); + } + assessment.getVulns().add(v); + } + + WordprocessingMLPackage mlp = WordprocessingMLPackage.createPackage(); + mlp.getMainDocumentPart().addParagraphOfText("Report intro"); + mlp.getMainDocumentPart().addParagraphOfText("${fiBegin}"); + mlp.getMainDocumentPart().addParagraphOfText("${vulnName}"); + mlp.getMainDocumentPart().addParagraphOfText("${desc}"); + mlp.getMainDocumentPart().addParagraphOfText("${rec}"); + mlp.getMainDocumentPart().addParagraphOfText("${details}"); + mlp.getMainDocumentPart().addParagraphOfText("${fiEnd}"); + mlp.getMainDocumentPart().addParagraphOfText(""); + mlp.getMainDocumentPart().addParagraphOfText("End of report"); + + EntityManagerFactory emf = Mockito.mock(EntityManagerFactory.class); + EntityManager em = Mockito.mock(EntityManager.class); + Query query = Mockito.mock(Query.class); + Mockito.when(emf.createEntityManager()).thenReturn(em); + Mockito.when(em.createQuery("from AppStore order by order")).thenReturn(query); + Mockito.when(query.getResultList()).thenReturn(new ArrayList()); + + MethodProfiler.setEnabled(true); + MethodProfiler.clearStats(); + + long start = System.currentTimeMillis(); + DocxUtils genDoc = new DocxUtils(emf, mlp, assessment); + genDoc.FONT = "Calibri"; + mlp = genDoc.generateDocx("p { margin: 0; } img { width: 50% !important; height: auto !important; }"); + long elapsed = System.currentTimeMillis() - start; + + System.out.println("\n=== Perf harness: vulns=" + vulnCount + " images=" + withImages + + " generateDocx wall=" + elapsed + "ms ==="); + MethodProfiler.printReport(); + MethodProfiler.clearStats(); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + mlp.save(baos); + System.out.println("Saved docx size: " + (baos.size() / 1024 / 1024) + " MB"); + } + + // screenshot-like PNG: dense text lines and UI blocks — compresses like + // a real terminal/browser capture and downscales the way real + // screenshots do (unlike random noise, which is incompressible) + private String makePng(Random rnd, int w, int h) throws Exception { + BufferedImage img = new BufferedImage(w, h, BufferedImage.TYPE_INT_RGB); + Graphics2D g = img.createGraphics(); + g.setColor(java.awt.Color.WHITE); + g.fillRect(0, 0, w, h); + for (int i = 0; i < 30; i++) { + g.setColor(new java.awt.Color(rnd.nextInt(0xFFFFFF))); + g.fillRect(rnd.nextInt(w), rnd.nextInt(h), rnd.nextInt(400) + 40, rnd.nextInt(60) + 10); + } + g.setFont(new java.awt.Font("Monospaced", java.awt.Font.PLAIN, 14)); + g.setColor(new java.awt.Color(40, 40, 40)); + for (int y = 14; y < h; y += 16) { + // realistic text coverage: most rows have a line of text of + // varying length, not wall-to-wall characters + if (rnd.nextInt(10) < 7) { + StringBuilder line = new StringBuilder(); + int len = rnd.nextInt(w / 16) + 8; + while (line.length() < len) { + line.append(Integer.toHexString(rnd.nextInt())).append(' '); + } + g.drawString(line.toString(), 8 + rnd.nextInt(40), y); + } + } + g.dispose(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ImageIO.write(img, "png", out); + return "data:image/png;base64," + Base64.getEncoder().encodeToString(out.toByteArray()); + } +} diff --git a/test/org/fuse/docx/DocxUtils2ParityTest.java b/test/org/fuse/docx/DocxUtils2ParityTest.java new file mode 100644 index 00000000..e338fa2a --- /dev/null +++ b/test/org/fuse/docx/DocxUtils2ParityTest.java @@ -0,0 +1,371 @@ +package org.fuse.docx; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Date; +import java.util.List; +import java.util.Random; + +import javax.persistence.EntityManager; +import javax.persistence.EntityManagerFactory; +import javax.persistence.Query; + +import org.docx4j.TextUtils; +import org.docx4j.openpackaging.packages.WordprocessingMLPackage; +import org.junit.Assume; +import org.junit.Test; +import org.mockito.Mockito; + +import com.fuse.dao.AppStore; +import com.fuse.dao.Assessment; +import com.fuse.dao.AssessmentType; +import com.fuse.dao.CustomField; +import com.fuse.dao.RiskLevel; +import com.fuse.dao.Teams; +import com.fuse.dao.User; +import com.fuse.dao.Vulnerability; +import com.fuse.reporting.DocxUtils; +import com.fuse.reporting.DocxUtils2; +import com.fuse.utils.MethodProfiler; + +/** + * Parity test harness for DocxUtils vs DocxUtils2. + * + * Runs both implementations against an identical template-package + + * assessment fixture and asserts that every vulnerability name and every + * HTML field (desc/rec/details) lands in the generated document with no + * leaked template markers. + * + * The default run uses a small fixture (6 vulns) so it stays under a second + * and exercises the core code paths. Set -Dfaction.parity.harness=true to + * also run a 400-vuln comparison that prints MethodProfiler reports side by + * side — this is how to compare wall time end-to-end before flipping + * traffic. + * + * Images are inline PNG data URIs (no `getImage?id=` references) so neither + * path touches MongoDB: image-resolving code is exercised against real + * bytes, but the test stays hermetic. + */ +public class DocxUtils2ParityTest { + + // ============================================================ + // Default test: small fixture, parity assertions + // ============================================================ + + @Test + public void parityBothImplementationsProduceAllFields() throws Exception { + int vulnCount = 6; + Assessment assessment = buildAssessment(vulnCount); + WordprocessingMLPackage template = buildTemplatePackage(); + + // snapshot the template to bytes so DocxUtils2 gets a byte-for-byte + // identical template after going through ZIP round-trip + ByteArrayOutputStream templateBytes = new ByteArrayOutputStream(); + template.save(templateBytes); + + // ----- run DocxUtils (the original) ----- + WordprocessingMLPackage mlpOrig = WordprocessingMLPackage.load(new ByteArrayInputStream(templateBytes.toByteArray())); + EntityManagerFactory emf = stubEntityManagerFactory(); + DocxUtils orig = new DocxUtils(emf, mlpOrig, assessment); + orig.FONT = "Calibri"; + MethodProfiler.setEnabled(true); + MethodProfiler.clearStats(); + long origStart = System.currentTimeMillis(); + mlpOrig = orig.generateDocx("p { margin: 0; }"); + long origElapsed = System.currentTimeMillis() - origStart; + ByteArrayOutputStream origBytes = new ByteArrayOutputStream(); + mlpOrig.save(origBytes); + String origText = extractText(mlpOrig); + + // ----- run DocxUtils2 (raw-XML path) ----- + DocxUtils2 gen2 = new DocxUtils2(emf, assessment); + gen2.FONT = "Calibri"; + MethodProfiler.clearStats(); + long gen2Start = System.currentTimeMillis(); + byte[] gen2Output = gen2.generateReport(new ByteArrayInputStream(templateBytes.toByteArray()), "p { margin: 0; }"); + long gen2Elapsed = System.currentTimeMillis() - gen2Start; + assertNotNull("DocxUtils2 output was null", gen2Output); + assertTrue("DocxUtils2 output was empty", gen2Output.length > 0); + WordprocessingMLPackage mlp2 = WordprocessingMLPackage.load(new ByteArrayInputStream(gen2Output)); + String gen2Text = extractText(mlp2); + + System.out.println("\n=== Parity test (vulns=" + vulnCount + ") ==="); + System.out.println("DocxUtils wall: " + origElapsed + "ms, size=" + origBytes.size() + "B, " + origText.length() + " chars"); + System.out.println("DocxUtils2 wall: " + gen2Elapsed + "ms, size=" + gen2Output.length + "B, " + gen2Text.length() + " chars"); + + // ----- assertions on every field per vulnerability ----- + for (int n = 1; n <= vulnCount; n++) { + assertTrue("DocxUtils missing vuln name " + n, origText.contains("Parity Issue " + n)); + assertTrue("DocxUtils missing desc " + n, origText.contains("ParityDesc" + n)); + assertTrue("DocxUtils missing rec " + n, origText.contains("ParityRec" + n)); + assertTrue("DocxUtils missing details " + n, origText.contains("ParityDetails" + n)); + } + for (int n = 1; n <= vulnCount; n++) { + assertTrue("DocxUtils2 missing vuln name " + n, gen2Text.contains("Parity Issue " + n)); + assertTrue("DocxUtils2 missing desc " + n, gen2Text.contains("ParityDesc" + n)); + assertTrue("DocxUtils2 missing rec " + n, gen2Text.contains("ParityRec" + n)); + assertTrue("DocxUtils2 missing details " + n, gen2Text.contains("ParityDetails" + n)); + } + + // ----- no leaked template markers in either output ----- + assertFalse("DocxUtils leaked ${fiBegin}", origText.contains("${fiBegin")); + assertFalse("DocxUtils leaked ${fiEnd}", origText.contains("${fiEnd")); + assertFalse("DocxUtils leaked ${vulnName}", origText.contains("${vulnName}")); + assertFalse("DocxUtils leaked ${desc}", origText.contains("${desc}")); + assertFalse("DocxUtils leaked ${rec}", origText.contains("${rec}")); + assertFalse("DocxUtils leaked ${details}", origText.contains("${details}")); + assertFalse("DocxUtils leaked ${loop", origText.contains("${loop")); + assertFalse("DocxUtils leaked ${asmtName}", origText.contains("${asmtName}")); + assertFalse("DocxUtils leaked split marker", origText.contains("FCT-FIELD-SPLIT")); + + assertFalse("DocxUtils2 leaked ${fiBegin}", gen2Text.contains("${fiBegin")); + assertFalse("DocxUtils2 leaked ${fiEnd}", gen2Text.contains("${fiEnd")); + assertFalse("DocxUtils2 leaked ${vulnName}", gen2Text.contains("${vulnName}")); + assertFalse("DocxUtils2 leaked ${desc}", gen2Text.contains("${desc}")); + assertFalse("DocxUtils2 leaked ${rec}", gen2Text.contains("${rec}")); + assertFalse("DocxUtils2 leaked ${details}", gen2Text.contains("${details}")); + assertFalse("DocxUtils2 leaked ${loop", gen2Text.contains("${loop")); + assertFalse("DocxUtils2 leaked ${asmtName}", gen2Text.contains("${asmtName}")); + assertFalse("DocxUtils2 leaked split marker", gen2Text.contains("FCT-FIELD-SPLIT")); + + // ----- assessment-level variables land in both outputs ----- + assertTrue("DocxUtils missing asmt name", origText.contains("Test PCI Assessment")); + assertTrue("DocxUtils2 missing asmt name", gen2Text.contains("Test PCI Assessment")); + assertTrue("DocxUtils missing asmt id", origText.contains("1337")); + assertTrue("DocxUtils2 missing asmt id", gen2Text.contains("1337")); + assertTrue("DocxUtils missing assessors", origText.contains("Bob Dobbs")); + assertTrue("DocxUtils2 missing assessors", gen2Text.contains("Bob Dobbs")); + assertTrue("DocxUtils missing riskTotal", origText.contains("6")); + assertTrue("DocxUtils2 missing riskTotal", gen2Text.contains("6")); + } + + // ============================================================ + // Perf harness: gated, 400-vuln comparison with profiler + // output for both implementations side by side. Run with + // -Dfaction.parity.harness=true -Dfaction.parity.vulns=400 + // ============================================================ + + @Test + public void parity400VulnsTimingComparison() throws Exception { + Assume.assumeTrue("parity perf harness disabled; pass -Dfaction.parity.harness=true to enable", + "true".equals(System.getProperty("faction.parity.harness"))); + int vulnCount = Integer.parseInt(System.getProperty("faction.parity.vulns", "400")); + + Assessment assessment = buildAssessment(vulnCount); + WordprocessingMLPackage template = buildTemplatePackage(); + ByteArrayOutputStream templateBytes = new ByteArrayOutputStream(); + template.save(templateBytes); + EntityManagerFactory emf = stubEntityManagerFactory(); + + // ----- DocxUtils (original) ----- + WordprocessingMLPackage mlpOrig = WordprocessingMLPackage.load(new ByteArrayInputStream(templateBytes.toByteArray())); + DocxUtils orig = new DocxUtils(emf, mlpOrig, assessment); + orig.FONT = "Calibri"; + MethodProfiler.setEnabled(true); + MethodProfiler.clearStats(); + long origStart = System.currentTimeMillis(); + mlpOrig = orig.generateDocx("p { margin: 0; } img { width: 50% !important; height: auto !important; }"); + long origElapsed = System.currentTimeMillis() - origStart; + long origSize = sizeOf(mlpOrig); + System.out.println("\n=== DocxUtils (original): vulns=" + vulnCount + " wall=" + origElapsed + "ms size=" + origSize + "B ==="); + MethodProfiler.printReport(); + MethodProfiler.clearStats(); + + // ----- DocxUtils2 (raw XML) ----- + DocxUtils2 gen2 = new DocxUtils2(emf, assessment); + gen2.FONT = "Calibri"; + MethodProfiler.setEnabled(true); + MethodProfiler.clearStats(); + long gen2Start = System.currentTimeMillis(); + byte[] gen2Output = gen2.generateReport(new ByteArrayInputStream(templateBytes.toByteArray()), + "p { margin: 0; } img { width: 50% !important; height: auto !important; }"); + long gen2Elapsed = System.currentTimeMillis() - gen2Start; + System.out.println("\n=== DocxUtils2 (raw XML): vulns=" + vulnCount + " wall=" + gen2Elapsed + "ms size=" + gen2Output.length + "B ==="); + MethodProfiler.printReport(); + MethodProfiler.clearStats(); + + System.out.println("\n=== Speedup: " + String.format("%.2fx", (double) origElapsed / Math.max(gen2Elapsed, 1)) + " ==="); + + // spot-check that the first and last vuln fields are present + String gen2Text; + { + WordprocessingMLPackage mlp2 = WordprocessingMLPackage.load(new ByteArrayInputStream(gen2Output)); + gen2Text = extractText(mlp2); + } + assertTrue("DocxUtils2 missing first vuln name", gen2Text.contains("Parity Issue 1")); + assertTrue("DocxUtils2 missing last vuln name", gen2Text.contains("Parity Issue " + vulnCount)); + assertFalse("DocxUtils2 leaked ${fiBegin", gen2Text.contains("${fiBegin")); + assertFalse("DocxUtils2 leaked ${desc", gen2Text.contains("${desc")); + } + + // ============================================================ + // Fixtures + // ============================================================ + + private Assessment buildAssessment(int vulnCount) { + Teams team = new Teams(); + team.setId(123L); + team.setTeamName("Hacking Team"); + AssessmentType type = new AssessmentType(); + type.setId(1235L); + type.setType("Assessment Type"); + + List levels = new ArrayList<>(); + String[] riskNames = { "Informational", "Recommended", "Low", "Medium", "High", "Critical" }; + for (int i = 0; i < 10; i++) { + RiskLevel level = new RiskLevel(); + level.setRiskId(i); + if (i < riskNames.length) { + level.setRisk(riskNames[i]); + } + levels.add(level); + } + + Assessment a = new Assessment(); + User u = new User(); + u.setId(1L); + u.setFname("Bob"); + u.setLname("Dobbs"); + u.setEmail("bdobs@supersecure.com"); + u.setTeam(team); + a.setName("Test PCI Assessment"); + a.setId(1337L); + a.setEngagement(u); + List hackers = new ArrayList<>(); + hackers.add(u); + hackers.add(u); + a.setType(type); + a.setAssessor(hackers); + a.setRemediation(u); + a.setAppId("1337"); + a.setRiskAnalysis("Risk analysis text content for parity check"); + a.setSummary("Summary section content for parity check"); + a.setVulns(new ArrayList<>()); + a.setStart(new Date()); + a.setEnd(new Date()); + a.setGuid("parity-guid"); + + Random rnd = new Random(42); + for (int i = 1; i <= vulnCount; i++) { + Vulnerability v = new Vulnerability(); + v.setLevels(levels); + v.setId((long) i); + v.setName("Parity Issue " + i); + v.setImpact((long) (i % 5) + 1); + v.setLikelyhood((long) (i % 5) + 1); + v.setOverall((long) (i % 5) + 1); + v.setCvssScore("8.3"); + v.setCvssString("CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:N/VC:H/VI:L/VA:L/SC:N/SI:N/SA:N"); + v.setTracking("VID-" + i); + v.setAssessmentId(1337L); + v.setCustomFields(new ArrayList()); + // Section is null so vulns land in Default — both implementations + // handle Default identically and resolvePlaceholderSpec in + // DocxUtils2 only knows about Default today + v.setSection(null); + // Distinct content per vuln so HTML conversion can't dedup + v.setDescription("ParityDesc" + i + " with a link and markup."); + v.setRecommendation("ParityRec" + i + "
  • fix it
  • test it
"); + // Half of the vulns embed a small inline PNG so the image path + // executes against real bytes without touching MongoDB + if (i % 2 == 0) { + v.setDetails("ParityDetails" + i + "
payload

" + + "
"); + } else { + v.setDetails("ParityDetails" + i + "
payload

plain text only"); + } + a.getVulns().add(v); + } + return a; + } + + /** + * Builds the template package using docx4j programmatic API. The + * template mirrors what real report templates contain: a findings + * block delimited by ${fiBegin}/${fiEnd} with per-vuln placeholders, + * followed by an assessment-level variable. + * + * Both implementations receive an identical byte-stream copy of this + * template — DocxUtils gets a freshly loaded WordprocessingMLPackage, + * DocxUtils2 gets the saved bytes back as an InputStream. + */ + private WordprocessingMLPackage buildTemplatePackage() throws Exception { + WordprocessingMLPackage mlp = WordprocessingMLPackage.createPackage(); + mlp.getMainDocumentPart().addParagraphOfText("Report intro"); + mlp.getMainDocumentPart().addParagraphOfText("${asmtName} assessment (${asmtId}) by ${asmtAssessor}"); + mlp.getMainDocumentPart().addParagraphOfText("${fiBegin}"); + mlp.getMainDocumentPart().addParagraphOfText("${vulnName}"); + mlp.getMainDocumentPart().addParagraphOfText("${desc}"); + mlp.getMainDocumentPart().addParagraphOfText("${rec}"); + mlp.getMainDocumentPart().addParagraphOfText("${details}"); + mlp.getMainDocumentPart().addParagraphOfText("${fiEnd}"); + mlp.getMainDocumentPart().addParagraphOfText(""); + mlp.getMainDocumentPart().addParagraphOfText("End of report"); + return mlp; + } + + private EntityManagerFactory stubEntityManagerFactory() { + EntityManagerFactory emf = Mockito.mock(EntityManagerFactory.class); + EntityManager em = Mockito.mock(EntityManager.class); + Query query = Mockito.mock(Query.class); + Mockito.when(emf.createEntityManager()).thenReturn(em); + Mockito.when(em.createQuery("from AppStore order by order")).thenReturn(query); + Mockito.when(query.getResultList()).thenReturn(new ArrayList()); + return emf; + } + + private static String extractText(WordprocessingMLPackage mlp) { + StringWriter sw = new StringWriter(); + try { + TextUtils.extractText(mlp.getMainDocumentPart().getContents(), sw); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + return sw.toString(); + } + + private static long sizeOf(WordprocessingMLPackage mlp) { + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + mlp.save(baos); + return baos.size(); + } catch (Exception ex) { + return -1; + } + } + + // tiny screenshot-like PNG so the image pipeline handles real bytes + private String makePng(Random rnd, int w, int h) { + try { + java.awt.image.BufferedImage img = new java.awt.image.BufferedImage(w, h, java.awt.image.BufferedImage.TYPE_INT_RGB); + java.awt.Graphics2D g = img.createGraphics(); + g.setColor(java.awt.Color.WHITE); + g.fillRect(0, 0, w, h); + g.setColor(new java.awt.Color(40, 40, 40)); + g.setFont(new java.awt.Font("Monospaced", java.awt.Font.PLAIN, 12)); + int y = 14; + while (y < h) { + StringBuilder line = new StringBuilder(); + int len = rnd.nextInt(w / 8) + 4; + while (line.length() < len) { + line.append(Integer.toHexString(rnd.nextInt())).append(' '); + } + g.drawString(line.toString(), 6, y); + y += 16; + } + g.dispose(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + javax.imageio.ImageIO.write(img, "png", out); + return "data:image/png;base64," + Base64.getEncoder().encodeToString(out.toByteArray()); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } +} diff --git a/test/org/fuse/docx/DocxUtilsBatchIntegrationTest.java b/test/org/fuse/docx/DocxUtilsBatchIntegrationTest.java new file mode 100644 index 00000000..2024fe73 --- /dev/null +++ b/test/org/fuse/docx/DocxUtilsBatchIntegrationTest.java @@ -0,0 +1,110 @@ +package org.fuse.docx; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; + +import javax.persistence.EntityManager; +import javax.persistence.EntityManagerFactory; +import javax.persistence.Query; + +import org.docx4j.TextUtils; +import org.docx4j.openpackaging.packages.WordprocessingMLPackage; +import org.junit.Test; +import org.mockito.Mockito; + +import com.fuse.dao.AppStore; +import com.fuse.dao.Assessment; +import com.fuse.dao.AssessmentType; +import com.fuse.dao.CustomField; +import com.fuse.dao.RiskLevel; +import com.fuse.dao.Teams; +import com.fuse.dao.Vulnerability; +import com.fuse.reporting.DocxUtils; +import com.fuse.reporting.GenerateReport; + +/** + * Exercises the batched XHTML conversion in setFindings end-to-end: many + * findings, multiple conversion chunks, and verifies every field lands in + * the generated document with no leaked markers or placeholders. + */ +public class DocxUtilsBatchIntegrationTest { + + @Test + public void batchedFindingsConversionProducesEveryField() throws Exception { + Teams team = new Teams(); + team.setId(123l); + team.setTeamName("Hacking Team"); + AssessmentType type = new AssessmentType(); + type.setId(1235l); + type.setType("Assessment Type"); + + List levels = new ArrayList<>(); + String[] risk = { "Informational", "Recommended", "Low", "Medium", "High", "Critical" }; + for (int i = 0; i < 10; i++) { + RiskLevel level = new RiskLevel(); + level.setRiskId(i); + if (i < risk.length) + level.setRisk(risk[i]); + levels.add(level); + } + + // 10 sections x 3 vulns = 30 findings -> 90+ fields -> several chunks + String[] sections = new String[10]; + for (int i = 0; i < sections.length; i++) { + sections[i] = "S" + i; + } + Assessment assessment = GenerateReport.createTestAssessment(team, type, levels, sections); + assessment.setGuid("test-guid"); + int i = 1; + for (Vulnerability v : assessment.getVulns()) { + v.setCustomFields(new ArrayList()); + v.setDescription("MarkerDescription" + i + " with a link"); + v.setRecommendation("MarkerRecommendation" + i + "
  • item one
  • item two
"); + v.setDetails("MarkerDetails" + i + " bold
code block
"); + i++; + } + int vulnCount = assessment.getVulns().size(); + + WordprocessingMLPackage mlp = WordprocessingMLPackage.createPackage(); + mlp.getMainDocumentPart().addParagraphOfText("${fiBegin}"); + mlp.getMainDocumentPart().addParagraphOfText("${vulnName}"); + mlp.getMainDocumentPart().addParagraphOfText("${desc}"); + mlp.getMainDocumentPart().addParagraphOfText("${rec}"); + mlp.getMainDocumentPart().addParagraphOfText("${details}"); + mlp.getMainDocumentPart().addParagraphOfText("${fiEnd}"); + // setFindings' template extraction consumes the element following + // ${fiEnd}; real templates always have trailing content + mlp.getMainDocumentPart().addParagraphOfText(""); + mlp.getMainDocumentPart().addParagraphOfText("End of report"); + + EntityManagerFactory emf = Mockito.mock(EntityManagerFactory.class); + EntityManager em = Mockito.mock(EntityManager.class); + Query query = Mockito.mock(Query.class); + Mockito.when(emf.createEntityManager()).thenReturn(em); + Mockito.when(em.createQuery("from AppStore order by order")).thenReturn(query); + Mockito.when(query.getResultList()).thenReturn(new ArrayList()); + + DocxUtils genDoc = new DocxUtils(emf, mlp, assessment); + genDoc.FONT = "Calibri"; + mlp = genDoc.generateDocx("p { margin: 0; }"); + + StringWriter text = new StringWriter(); + TextUtils.extractText(mlp.getMainDocumentPart().getContents(), text); + String docText = text.toString(); + + for (int n = 1; n <= vulnCount; n++) { + assertTrue("missing vuln name " + n, docText.contains("Test Issue " + n)); + assertTrue("missing desc " + n, docText.contains("MarkerDescription" + n)); + assertTrue("missing rec " + n, docText.contains("MarkerRecommendation" + n)); + assertTrue("missing details " + n, docText.contains("MarkerDetails" + n)); + } + assertFalse("split marker leaked into document", docText.contains("FCT-FIELD-SPLIT")); + assertFalse("unresolved rec placeholder", docText.contains("${rec")); + assertFalse("unresolved desc placeholder", docText.contains("${desc")); + assertFalse("unresolved details placeholder", docText.contains("${details")); + } +} diff --git a/test/org/fuse/docx/ReportImageScalerTest.java b/test/org/fuse/docx/ReportImageScalerTest.java new file mode 100644 index 00000000..a2f5e4d7 --- /dev/null +++ b/test/org/fuse/docx/ReportImageScalerTest.java @@ -0,0 +1,145 @@ +package org.fuse.docx; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.Base64; + +import javax.imageio.ImageIO; + +import org.junit.Test; + +import com.fuse.utils.FSUtils; +import com.fuse.utils.ReportImageScaler; + +public class ReportImageScalerTest { + + private String makePng(int w, int h) throws Exception { + BufferedImage img = new BufferedImage(w, h, BufferedImage.TYPE_INT_RGB); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ImageIO.write(img, "png", out); + return "data:image/png;base64," + Base64.getEncoder().encodeToString(out.toByteArray()); + } + + @Test + public void downscalesWidePng() throws Exception { + String uri = makePng(2400, 1400); + String scaled = ReportImageScaler.downscaleDataUri(uri, 1600); + assertNotEquals("should have been rewritten", uri, scaled); + byte[] bytes = Base64.getMimeDecoder().decode(scaled.substring(scaled.indexOf(',') + 1)); + BufferedImage img = ImageIO.read(new ByteArrayInputStream(bytes)); + assertEquals(1600, img.getWidth()); + } + + @Test + public void leavesSmallPngAlone() throws Exception { + String uri = makePng(800, 500); + assertEquals(uri, ReportImageScaler.downscaleDataUri(uri, 1600)); + } + + @Test + public void inlineUriSurvivesJtidyUnchanged() throws Exception { + String uri = makePng(1200, 700); + String content = "Details text

"; + String tidied = FSUtils.jtidy(content); + assertTrue("jtidy must preserve the data URI byte-for-byte, or the " + + "warm-cache hash lookup misses", tidied.contains(uri)); + } + + @Test + public void megabyteUriSurvivesJtidyUnchanged() throws Exception { + // realistic screenshot-scale URI (~1MB+); jtidy behavior can differ + // with huge attribute values + java.awt.image.BufferedImage img = new java.awt.image.BufferedImage(2400, 1400, + java.awt.image.BufferedImage.TYPE_INT_RGB); + java.awt.Graphics2D g = img.createGraphics(); + g.setColor(java.awt.Color.WHITE); + g.fillRect(0, 0, 2400, 1400); + java.util.Random rnd = new java.util.Random(7); + g.setFont(new java.awt.Font("Monospaced", java.awt.Font.PLAIN, 14)); + for (int y = 14; y < 1400; y += 16) { + g.setColor(new java.awt.Color(rnd.nextInt(64), rnd.nextInt(64), rnd.nextInt(64))); + StringBuilder line = new StringBuilder(); + while (line.length() < 300) { + line.append(Integer.toHexString(rnd.nextInt())).append(' '); + } + g.drawString(line.toString(), 4, y); + } + g.dispose(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ImageIO.write(img, "png", out); + String uri = "data:image/png;base64," + Base64.getEncoder().encodeToString(out.toByteArray()); + System.out.println("URI length: " + uri.length()); + + String scaled = ReportImageScaler.downscaleDataUri(uri, 1600); + System.out.println("scaled length: " + scaled.length() + " changed=" + !scaled.equals(uri) + + " mime=" + scaled.substring(0, scaled.indexOf(';'))); + + // exercise the jpeg encoder directly via reflection to see its size + java.lang.reflect.Method m = ReportImageScaler.class.getDeclaredMethod("encodeJpeg", + java.awt.image.BufferedImage.class, float.class); + m.setAccessible(true); + byte[] raw = Base64.getMimeDecoder().decode(uri.substring(uri.indexOf(',') + 1)); + java.awt.image.BufferedImage src = ImageIO.read(new ByteArrayInputStream(raw)); + byte[] jpg = (byte[]) m.invoke(null, src, 0.85f); + System.out.println("direct jpeg bytes: " + (jpg == null ? "null" : jpg.length) + + " original bytes: " + raw.length); + assertNotEquals("scaler should shrink a text-style screenshot", uri, scaled); + + String content = "Details text

"; + String tidied = FSUtils.jtidy(content); + assertTrue("jtidy altered a megabyte data URI", tidied.contains(uri)); + } + + @Test + public void prepareStoresRenditionForOversizedImage() throws Exception { + com.fuse.dao.Image img = new com.fuse.dao.Image(); + img.setBase64Image(makePng(2400, 1400)); + int maxWidth = ReportImageScaler.configuredMaxWidth(); + + assertTrue("first prepare must modify the entity", ReportImageScaler.prepareReportRendition(img)); + assertEquals(Integer.valueOf(maxWidth), img.getReportWidth()); + assertNotEquals("oversized image must get a distinct rendition", null, img.getReportImage()); + assertTrue(ReportImageScaler.isReportReady(img, maxWidth)); + + // the rendition is what report generation embeds, with no decode work + assertEquals(img.getReportImage(), ReportImageScaler.reportUri(img, maxWidth)); + + // idempotent: a second prepare for the same cap is a no-op + assertTrue("re-prepare for the same cap must be a no-op", + !ReportImageScaler.prepareReportRendition(img)); + } + + @Test + public void prepareMarksSmallImageWithoutDuplicatingIt() throws Exception { + com.fuse.dao.Image img = new com.fuse.dao.Image(); + String uri = makePng(800, 500); + img.setBase64Image(uri); + int maxWidth = ReportImageScaler.configuredMaxWidth(); + + assertTrue(ReportImageScaler.prepareReportRendition(img)); + assertEquals("already-small image must not be stored twice", null, img.getReportImage()); + assertEquals(Integer.valueOf(maxWidth), img.getReportWidth()); + // reportUri serves the original directly + assertEquals(uri, ReportImageScaler.reportUri(img, maxWidth)); + } + + @Test + public void staleWidthFallsBackToLiveDownscale() throws Exception { + com.fuse.dao.Image img = new com.fuse.dao.Image(); + img.setBase64Image(makePng(2400, 1400)); + img.setReportImage("data:image/png;base64,STALE"); + img.setReportWidth(999); // prepared for a different cap + + assertTrue(!ReportImageScaler.isReportReady(img, 1600)); + String served = ReportImageScaler.reportUri(img, 1600); + assertNotEquals("stale rendition must not be served", img.getReportImage(), served); + byte[] bytes = Base64.getMimeDecoder().decode(served.substring(served.indexOf(',') + 1)); + BufferedImage decoded = ImageIO.read(new ByteArrayInputStream(bytes)); + assertEquals("must be a live downscale of the original", 1600, decoded.getWidth()); + } +}