Version: 2.5.8 Update

.github/ISSUE_TEMPLATE/问题反馈.md

@@ -0,0 +1,27 @@
+---
+name: 问题反馈
+about: 尽可能详细的描述问题并反馈
+title: "[BUG] "
+labels: bug
+assignees: ''
+
+---
+
+## 使用环境
+
+```
+HaE版本：
+BurpSuite版本：
+JDK版本：
+操作系统版本：
+```
+
+## 问题详情
+
+问题描述：
+
+出现的场景：
+
+## 解决建议
+
+无。

README.md

@@ -23,6 +23,22 @@
 
 除此之外，您也可以选择将配置文件存放在`HaE Jar包`的同级目录下的`/.config/HaE/`中，**以便于离线携带**。
 
+### 规则释义
+
+HaE目前的规则一共有6个字段，分别是规则名称、规则正则、规则作用域、正则引擎、规则匹配颜色、规则敏感性。
+
+详细的含义如下所示：
+
+| 字段      | 含义                                                                                                                                                                                                   |
+|-----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Name      | 规则名称，主要用于简短概括当前规则的作用。                                                                                                                                                               |
+| Regex     | 规则正则，主要用于填写正则表达式。在HaE中所需提取匹配的内容需要用`(`、`)`将正则表达式进行包裹。                                                                                                            |
+| Scope     | 规则作用域，主要用于表示当前规则作用与HTTP报文的哪个部分。                                                                                                                                               |
+| Engine    | 正则引擎，主要用于表示当前规则的正则表达式所使用的引擎。**DFA引擎**：对于文本串里的每一个字符只需扫描一次，速度快、特性少；**NFA引擎**：要翻来覆去标注字符、取消标注字符，速度慢，但是特性（如:分组、替换、分割）丰富。 |
+| Color     | 规则匹配颜色，主要用于表示当前规则匹配到对应HTTP报文时所需标记的高亮颜色。                                                                                                                               |
+| Sensitive | 规则敏感性，主要用于表示当前规则对于大小写字母是否敏感，敏感（`True`）则严格按照大小写要求匹配，不敏感（`False`）则反之。                                                                                      |
+
+
 ## 优势特点
 
 1. 精细配置：高度自由的配置选项，以满足各类精细化场景需求。

src/main/java/burp/BurpExtender.java

@@ -9,7 +9,6 @@ import burp.ui.board.MessagePanel;
 import java.beans.PropertyChangeEvent;
 import java.beans.PropertyChangeListener;
 import java.net.URL;
-import java.security.NoSuchAlgorithmException;
 import java.util.*;
 import javax.swing.*;
 import java.awt.*;
@@ -39,7 +38,7 @@ public class BurpExtender implements IBurpExtender, IHttpListener, IMessageEdito
 
         new ConfigLoader();
 
-        String version = "2.5.7";
+        String version = "2.5.8";
         callbacks.setExtensionName(String.format("HaE (%s) - Highlighter and Extractor", version));
 
         // 定义输出
@@ -121,65 +120,41 @@ public class BurpExtender implements IBurpExtender, IHttpListener, IMessageEdito
     public void processHttpMessage(int toolFlag, boolean messageIsRequest, IHttpRequestResponse messageInfo) {
         // 判断是否是响应，且该代码作用域为：REPEATER、INTRUDER、PROXY（分别对应toolFlag 64、32、4）
         if (toolFlag == 64 || toolFlag == 32 || toolFlag == 4) {
-            byte[] content;
+            if (!messageIsRequest) {
+                IHttpService iHttpService = messageInfo.getHttpService();
+                String host = iHttpService.getHost();
 
-            if (messageIsRequest) {
-                content = messageInfo.getRequest();
-            } else {
-                content = messageInfo.getResponse();
-            }
+                List<Map<String, String>> result = null;
 
-            IHttpService iHttpService = null;
+                String originalColor = messageInfo.getHighlight();
+                String originalComment = messageInfo.getComment();
 
-            String host = "";
+                try {
+                    result = messageProcessor.processMessage(helpers, messageInfo, host, true);
 
-            try {
-                iHttpService = messageInfo.getHttpService();
-                host = iHttpService.getHost();
-            } catch (Exception ignored) {
-            }
+                    if (result != null && !result.isEmpty() && result.size() > 0) {
+                        List<String> colorList = new ArrayList<>();
 
-            if (Objects.equals(host, "")) {
-                List<String> requestTmpHeaders = helpers.analyzeRequest(content).getHeaders();
-                host = requestTmpHeaders.get(1).split(":")[1].trim();
-            }
+                        if (originalColor != null) {
+                            colorList.add(originalColor);
+                        }
 
-            List<Map<String, String>> result = null;
+                        colorList.add(result.get(0).get("color"));
+                        String resColor = colorProcessor.retrieveFinalColor(colorProcessor.retrieveColorIndices(colorList));
+                        messageInfo.setHighlight(resColor);
 
-            try {
-                result = messageProcessor.processMessage(helpers, content, messageIsRequest, true, host);
-            } catch (NoSuchAlgorithmException e) {
-                throw new RuntimeException(e);
-            }
+                        String addComment = String.join(", ", result.get(1).get("comment"));
+                        String allComment = !Objects.equals(originalComment, "") ? String.format("%s, %s", originalComment, addComment) : addComment;
+                        String resComment = mergeComment(allComment);
+                        messageInfo.setComment(resComment);
 
-            String resComment = "";
-            String resColor = "";
-            String originalColor = messageInfo.getHighlight();
-            String originalComment = messageInfo.getComment();
-
-            if (result != null && !result.isEmpty() && result.size() > 0) {
-                List<String> colorList = new ArrayList<>();
-
-                if (originalColor != null) {
-                    colorList.add(originalColor);
+                        messagePanel.add(messageInfo, resComment, resColor);
+                    }
+                } catch (Exception e) {
+                    e.printStackTrace();
                 }
-
-                colorList.add(result.get(0).get("color"));
-                resColor = colorProcessor.retrieveFinalColor(colorProcessor.retrieveColorIndices(colorList));
-                messageInfo.setHighlight(resColor);
-
-                String addComment = String.join(", ", result.get(1).get("comment"));
-                String allComment = !Objects.equals(originalComment, "") ? String.format("%s, %s", originalComment, addComment) : addComment;
-                resComment = mergeComment(allComment);
-                messageInfo.setComment(resComment);
             }
 
-            String endComment = resComment.isEmpty() ? originalComment : resComment;
-            String endColor = resColor.isEmpty() ? originalColor : resColor;
-
-            if (!messageIsRequest && !Objects.equals(endComment, "") && !Objects.equals(endColor, "")) {
-                messagePanel.add(messageInfo, endComment, String.valueOf(content.length), endColor);
-            }
         }
     }
 
@@ -250,9 +225,13 @@ public class BurpExtender implements IBurpExtender, IHttpListener, IMessageEdito
             List<Map<String, String>> result = null;
 
             try {
-                result = messageProcessor.processMessage(helpers, content, isRequest, false, "");
-            } catch (NoSuchAlgorithmException e) {
-                throw new RuntimeException(e);
+                if (isRequest) {
+                    result = messageProcessor.processRequestMessage(helpers, content, "", false);
+                } else {
+                    result = messageProcessor.processResponseMessage(helpers, content, "", false);
+                }
+            } catch (Exception e) {
+                e.printStackTrace();
             }
 
             if (result != null && !result.isEmpty()) {
@@ -264,6 +243,7 @@ public class BurpExtender implements IBurpExtender, IHttpListener, IMessageEdito
                 }
                 return true;
             }
+
             return false;
         }
 

src/main/java/burp/core/processor/DataProcessingUnit.java

@@ -132,7 +132,7 @@ public class DataProcessingUnit {
                             tmpMap.put("data", dataStr);
                             finalMap.put(nameAndSize, tmpMap);
                             // 添加到全局变量中，便于Databoard检索
-                            if (!Objects.equals(host, "")) {
+                            if (!Objects.equals(host, "") && host != null) {
                                 List<String> dataList = Arrays.asList(dataStr.split("\n"));
                                 if (ConfigEntry.globalDataMap.containsKey(host)) {
                                     Map<String, List<String>> gRuleMap = new HashMap<>(ConfigEntry.globalDataMap.get(host));

src/main/java/burp/core/processor/MessageProcessor.java

@@ -1,10 +1,10 @@
 package burp.core.processor;
 
 import burp.IExtensionHelpers;
+import burp.IHttpRequestResponse;
 import burp.IRequestInfo;
 import burp.IResponseInfo;
 import burp.core.utils.MatchTool;
-import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -12,53 +12,78 @@ import java.util.List;
 import java.util.Map;
 
 public class MessageProcessor {
-    MatchTool matcher = new MatchTool();
-    DataProcessingUnit dataProcessingUnit = new DataProcessingUnit();
-    ColorProcessor colorProcessor = new ColorProcessor();
+    private MatchTool matcher = new MatchTool();
+    private DataProcessingUnit dataProcessingUnit = new DataProcessingUnit();
+    private ColorProcessor colorProcessor = new ColorProcessor();
 
-    public List<Map<String, String>> processMessage(IExtensionHelpers helpers, byte[] content, boolean isRequest, boolean messageInfo, String host)
-            throws NoSuchAlgorithmException {
-        List<Map<String, String>> result = new ArrayList<>();
+    public List<Map<String, String>> processMessage(IExtensionHelpers helpers, IHttpRequestResponse messageInfo, String host, boolean actionFlag) throws Exception {
+
+        byte[] requestByte = messageInfo.getRequest();
+        byte[] responseByte = messageInfo.getResponse();
+
+        List<Map<String, String>> reqObj = processRequestMessage(helpers, requestByte, host, actionFlag);
+        List<Map<String, String>> resObj = processResponseMessage(helpers, responseByte, host, actionFlag);
+
+        List<Map<String, String>> mergedList = new ArrayList<>(reqObj);
+        mergedList.addAll(resObj);
+
+        return mergedList;
+    }
+
+    public List<Map<String, String>> processRequestMessage(IExtensionHelpers helpers, byte[] content, String host, boolean actionFlag) throws Exception {
         Map<String, Map<String, Object>> obj;
 
-        if (isRequest) {
-            IRequestInfo requestInfo = helpers.analyzeRequest(content);
-            List<String> requestTmpHeaders = requestInfo.getHeaders();
-            String requestHeaders = String.join("\n", requestTmpHeaders);
+        IRequestInfo requestInfo = helpers.analyzeRequest(content);
+        List<String> requestTmpHeaders = requestInfo.getHeaders();
+        String requestHeaders = String.join("\n", requestTmpHeaders);
 
-            try {
-                String urlString = requestTmpHeaders.get(0).split(" ")[1];
-                urlString = urlString.indexOf("?") > 0 ? urlString.substring(0, urlString.indexOf("?")) : urlString;
-                if (matcher.matchUrlSuffix(urlString)) {
-                    return result;
-                }
-            } catch (Exception e) {
-                return result;
+        try {
+            String urlString = requestTmpHeaders.get(0).split(" ")[1];
+            urlString = urlString.indexOf("?") > 0 ? urlString.substring(0, urlString.indexOf("?")) : urlString;
+            if (matcher.matchUrlSuffix(urlString)) {
+                return null;
             }
-
-            int requestBodyOffset = requestInfo.getBodyOffset();
-            byte[] requestBody = Arrays.copyOfRange(content, requestBodyOffset, content.length);
-            obj = dataProcessingUnit.matchContentByRegex(content, requestHeaders, requestBody, "request", host);
-        } else {
-            IResponseInfo responseInfo = helpers.analyzeResponse(content);
-            try {
-                String inferredMimeType = String.format("hae.%s", responseInfo.getInferredMimeType().toLowerCase());
-                String statedMimeType = String.format("hae.%s", responseInfo.getStatedMimeType().toLowerCase());
-                if (matcher.matchUrlSuffix(statedMimeType) || matcher.matchUrlSuffix(inferredMimeType)) {
-                    return result;
-                }
-            } catch (Exception e) {
-                return result;
-            }
-            List<String> responseTmpHeaders = responseInfo.getHeaders();
-            String responseHeaders = String.join("\n", responseTmpHeaders);
-            int responseBodyOffset = responseInfo.getBodyOffset();
-            byte[] responseBody = Arrays.copyOfRange(content, responseBodyOffset, content.length);
-            obj = dataProcessingUnit.matchContentByRegex(content, responseHeaders, responseBody, "response", host);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return null;
         }
 
+        int requestBodyOffset = requestInfo.getBodyOffset();
+        byte[] requestBody = Arrays.copyOfRange(content, requestBodyOffset, content.length);
+        obj = dataProcessingUnit.matchContentByRegex(content, requestHeaders, requestBody, "request", host);
+
+        return getDataList(obj, actionFlag);
+    }
+
+    public List<Map<String, String>> processResponseMessage(IExtensionHelpers helpers, byte[] content, String host, boolean actionFlag) throws Exception {
+        Map<String, Map<String, Object>> obj;
+
+        IResponseInfo responseInfo = helpers.analyzeResponse(content);
+        try {
+            String inferredMimeType = String.format("hae.%s", responseInfo.getInferredMimeType().toLowerCase());
+            String statedMimeType = String.format("hae.%s", responseInfo.getStatedMimeType().toLowerCase());
+            if (matcher.matchUrlSuffix(statedMimeType) || matcher.matchUrlSuffix(inferredMimeType)) {
+                return null;
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            return null;
+        }
+        List<String> responseTmpHeaders = responseInfo.getHeaders();
+        String responseHeaders = String.join("\n", responseTmpHeaders);
+        int responseBodyOffset = responseInfo.getBodyOffset();
+        byte[] responseBody = Arrays.copyOfRange(content, responseBodyOffset, content.length);
+        obj = dataProcessingUnit.matchContentByRegex(content, responseHeaders, responseBody, "response", host);
+
+        return getDataList(obj, actionFlag);
+    }
+
+    private List<Map<String, String>> getDataList(Map<String, Map<String, Object>> obj, boolean actionFlag) {
+        List<Map<String, String>> highlightList = new ArrayList<>();
+        List<Map<String, String>> extractList = new ArrayList<>();
+
         if (obj.size() > 0) {
-            if (messageInfo) {
+            if (actionFlag) {
                 List<List<String>> resultList = dataProcessingUnit.extractColorsAndComments(obj);
                 List<String> colorList = resultList.get(0);
                 List<String> commentList = resultList.get(1);
@@ -70,13 +95,14 @@ public class MessageProcessor {
                     Map<String, String> commentMap = new HashMap<String, String>() {{
                         put("comment", String.join(", ", commentList));
                     }};
-                    result.add(colorMap);
-                    result.add(commentMap);
+                    highlightList.add(colorMap);
+                    highlightList.add(commentMap);
                 }
             } else {
-                result.add(dataProcessingUnit.extractDataFromMap(obj));
+                extractList.add(dataProcessingUnit.extractDataFromMap(obj));
             }
         }
-        return result;
+
+        return actionFlag ? highlightList : extractList;
     }
 }

src/main/java/burp/rule/utils/RuleTool.java

@@ -1,8 +1,6 @@
 package burp.rule.utils;
 
 import burp.*;
-import burp.config.ConfigEntry;
-import burp.config.ConfigLoader;
 import java.io.FileOutputStream;
 import java.net.URL;
 import java.util.Arrays;

src/main/java/burp/ui/board/Databoard.java

@@ -301,9 +301,6 @@ public class Databoard extends JPanel {
                 for (Map.Entry<String, Map<String, List<String>>> entry : dataMap.entrySet()) {
                     JTabbedPane newTabbedPane = new JTabbedPane();
                     newTabbedPane.setTabLayoutPolicy(JTabbedPane.SCROLL_TAB_LAYOUT);
-                    if (currentWorker != null && !currentWorker.isDone()) {
-                        currentWorker.cancel(true);
-                    }
 
                     for (Map.Entry<String, List<String>> entrySet : entry.getValue().entrySet()) {
                         currentWorker = new SwingWorker<Object, Void>() {
@@ -322,8 +319,10 @@ public class Databoard extends JPanel {
                                 if (!isCancelled()) {
                                     try {
                                         Object[] result = (Object[]) get();
-                                        newTabbedPane.addTab(result[0].toString(), (DatatablePanel) result[1]);
-                                        dataTabbedPane.addTab(entry.getKey(), newTabbedPane);
+                                        SwingUtilities.invokeLater(() -> {
+                                            newTabbedPane.addTab(result[0].toString(), (DatatablePanel) result[1]);
+                                            dataTabbedPane.addTab(entry.getKey(), newTabbedPane);
+                                        });
                                     } catch (Exception e) {
                                         e.printStackTrace();
                                     }

src/main/java/burp/ui/board/MessagePanel.java

@@ -308,13 +308,13 @@ public class MessagePanel extends AbstractTableModel implements IMessageEditorCo
         return currentlyDisplayedItem.getHttpService();
     }
 
-    public void add(IHttpRequestResponse messageInfo, String comment, String length, String color) {
+    public void add(IHttpRequestResponse messageInfo, String comment, String color) {
         synchronized(log) {
             IRequestInfo iRequestInfo = helpers.analyzeRequest(messageInfo);
             URL url = iRequestInfo.getUrl();
             String method = iRequestInfo.getMethod();
             String status = String.valueOf(helpers.analyzeResponse(messageInfo.getResponse()).getStatusCode());
-
+            String length = String.valueOf(messageInfo.getResponse().length);
             LogEntry logEntry = new LogEntry(callbacks.saveBuffersToTempFiles(messageInfo), method, url, comment, length, color, status);
 
             try {
@@ -329,8 +329,8 @@ public class MessagePanel extends AbstractTableModel implements IMessageEditorCo
                         byte[] reqByteB = reqResMessage.getRequest();
                         byte[] resByteB = reqResMessage.getResponse();
                         try {
-                            // 采用匹配数据结果比对
-                            if (areMapsEqual(getCacheData(reqByteB), getCacheData(reqByteA)) && areMapsEqual(getCacheData(resByteB), getCacheData(resByteA))) {
+                            // 通过URL、请求和响应报文、匹配数据内容，多维度进行对比
+                            if ((entry.getUrl().toString().equals(url.toString()) || (Arrays.equals(reqByteB, reqByteA) || Arrays.equals(resByteB, resByteA))) && (areMapsEqual(getCacheData(reqByteB), getCacheData(reqByteA)) && areMapsEqual(getCacheData(resByteB), getCacheData(resByteA)))) {
                                 isDuplicate = true;
                                 break;
                             }