Merge pull request #7 from gh0stkey/master

HaE 1.5.1

.gitignore

@@ -1,2 +1 @@
-.gradle/
-build/
+.DS_Store

README.md

@@ -41,7 +41,7 @@ HaE supports three actions:
 3. Color upgrade algorithm: **Two regulars expression, the colors are both orange, if the request are matched these, it will be upgraded to red.**
 4. The configuration file format uses JSON format, the format is
     ```
-    {name: {"loaded": isLoaded,"regex": regexText, "scope": request/response/any, "action": extract/highlight/any, "color": colorText}}
+    {name: {"loaded": isLoaded,"regex": regexText, "scope": request/response/any, "action": extract/highlight/any, "color": colorText, "engine": dfa/nfa}}
     ```
 5. Built-in simple cache to reduce the stuttering phenomenon in the `multi-regular, big data scenario`.
 

README_zh.md

@@ -37,7 +37,7 @@ HaE支持三个动作:
 3. 颜色升级算法: 利用下标的方式进行优先级排序，当满足2个同颜色条件则以优先级顺序上升颜色。（例如: **两个正则，颜色为橘黄色，该请求两个正则都匹配到了，那么将升级为红色**）
 4. 简单的配置文件格式选用JSON格式，格式为
     ```
-    {name: {"loaded": isLoaded,"regex": regexText, "scope": request/response/any, "action": extract/highlight/any, "color": colorText}}
+    {name: {"loaded": isLoaded,"regex": regexText, "scope": request/response/any, "action": extract/highlight/any, "color": colorText, "engine": dfa/nfa}}
     ```
 5. 内置简单缓存，在“多正则、大数据”的场景下减少卡顿现象。
 

build.gradle

@@ -8,6 +8,7 @@ dependencies {
     compile 'net.portswigger.burp.extender:burp-extender-api:1.7.13'
     compile 'net.sourceforge.jregex:jregex:1.2_01'
     compile 'org.json:json:20200518'
+    compile 'dk.brics.automaton:automaton:1.11-8'
 }
 
 sourceSets {

burp/BurpExtender.java

@@ -72,16 +72,16 @@ public class BurpExtender implements IBurpExtender, IHttpListener, IMessageEdito
     public void registerExtenderCallbacks(final IBurpExtenderCallbacks callbacks)
     {
     	this.callbacks = callbacks;
-    this.helpers = callbacks.getHelpers();
+    BurpExtender.helpers = callbacks.getHelpers();
         // 设置插件名字和版本
-    	String version = "1.5";
+    	String version = "1.5.1";
 
         callbacks.setExtensionName(String.format("HaE (%s) - Highlighter and Extractor", version));
         
         // 定义输出
         stdout = new PrintWriter(callbacks.getStdout(), true);
         stdout.println("@Author: EvilChen");
-        stdout.println("@Blog: cn.gh0st.cn");
+        stdout.println("@Blog: gh0st.cn");
 
         // UI
 		SwingUtilities.invokeLater(new Runnable() {
@@ -174,7 +174,10 @@ public class BurpExtender implements IBurpExtender, IHttpListener, IMessageEdito
 				rules.add("red");
 				rules.add("response");
 				rules.add("any");
+				rules.add("nfa");
 				dtm.addRow(rules);
+				// 新增之后刷新Table，防止存在未刷新删除导致错位
+				ft.fillTable(configFilePath, table);
 			}
 		});
 		panel_1.add(btnNewRule);
@@ -206,7 +209,7 @@ public class BurpExtender implements IBurpExtender, IHttpListener, IMessageEdito
 			new Object[][] {
 			},
 			new String[] {
-				"Loaded", "Name", "Regex", "Color", "Scope", "Action"
+				"Loaded", "Name", "Regex", "Color", "Scope", "Action", "Engine"
 			}
 		));
 		scrollPane.setViewportView(table);
@@ -216,6 +219,7 @@ public class BurpExtender implements IBurpExtender, IHttpListener, IMessageEdito
 		table.getColumnModel().getColumn(0).setCellEditor(new DefaultCellEditor(new JCheckBox()));
 		table.getColumnModel().getColumn(4).setCellEditor(new DefaultCellEditor(new JComboBox(Config.scopeArray)));
 		table.getColumnModel().getColumn(5).setCellEditor(new DefaultCellEditor(new JComboBox(Config.actionArray)));
+		table.getColumnModel().getColumn(6).setCellEditor(new DefaultCellEditor(new JComboBox(Config.engineArray)));
 		
 		JLabel lblNewLabel = new JLabel("@EvilChen Love YuChen.");
 		lblNewLabel.setHorizontalAlignment(SwingConstants.CENTER);
@@ -237,6 +241,7 @@ public class BurpExtender implements IBurpExtender, IHttpListener, IMessageEdito
 			    			jsonObj1.put("color", (String) dtm.getValueAt(i, 3));
 			    			jsonObj1.put("scope", (String) dtm.getValueAt(i, 4));
 			    			jsonObj1.put("action", (String) dtm.getValueAt(i, 5));
+			    			jsonObj1.put("engine", (String) dtm.getValueAt(i, 6));
 			    			// 添加数据
 			    			jsonObj.put((String) dtm.getValueAt(i, 1), jsonObj1);
 						}
@@ -290,11 +295,6 @@ public class BurpExtender implements IBurpExtender, IHttpListener, IMessageEdito
 				return;
 			}
 	        if (messageIsRequest) {
-	            try {
-					String c = new String(content, "UTF-8").intern();
-				} catch (UnsupportedEncodingException e) {
-					e.printStackTrace();
-				}
 	            jsonObj = ec.matchRegex(content, "request", "highlight", configFilePath); 
 	        } else {
 	            content = messageInfo.getResponse();
@@ -304,11 +304,6 @@ public class BurpExtender implements IBurpExtender, IHttpListener, IMessageEdito
 	            if (mh.matchMIME(mimeList)) {
 					return;
 				}
-	            try {
-					String c = new String(content, "UTF-8").intern();
-				} catch (UnsupportedEncodingException e) {
-					e.printStackTrace();
-				}
 	            jsonObj = ec.matchRegex(content, "response", "highlight", configFilePath); 
 	        }
 	        
@@ -358,7 +353,6 @@ public class BurpExtender implements IBurpExtender, IHttpListener, IMessageEdito
 				return false;
 			}
 			
-            
 			if (isRequest) {
 				JSONObject jsonObj = ec.matchRegex(content, "request", "extract", configFilePath);
 				if (jsonObj.length() != 0) {

burp/Config.java

@@ -1,11 +1,16 @@
 package burp;
 
 public class Config {
-	public static String initConfigContent = "{\"Email\":{\"loaded\":true,\"scope\":\"response\",\"regex\":\"([\\\\w-]+(?:\\\\.[\\\\w-]+)*@(?:[\\\\w](?:[\\\\w-]*[\\\\w])?\\\\.)+[\\\\w](?:[\\\\w-]*[\\\\w])?)\",\"action\":\"any\",\"color\":\"yellow\"}}";
+	public static String initConfigContent = "{\"Email\":{\"loaded\":true,\"scope\":\"response\",\"regex\":\"([\\\\w-]+(?:\\\\.[\\\\w-]+)*@(?:[\\\\w](?:[\\\\w-]*[\\\\w])?\\\\.)+[\\\\w](?:[\\\\w-]*[\\\\w])?)\",\"action\":\"any\",\"color\":\"yellow\",\"engine\":\"nfa\"}}";
+	
 	public static String[] colorArray = new String[] {"red", "orange", "yellow", "green", "cyan", "blue", "pink", "magenta", "gray"};
 	public static String[] scopeArray = new String[] {"any", "response", "request"};
 	public static String[] actionArray = new String[] {"any", "extract", "highight"};
-	public static String excludeSuffix = "7z|aif|aifc|aiff|au|bmp|cmx|cod|css|doc|docx|gif|gz|ico|ief|jfif|jpe|jpeg|jpg|m3u|mid|mp2|mp3|mpa|mpe|mpeg|mpg|mpp|mpv2|otf|pbm|pdf|pgm|png|pnm|ppm|ra|ram|rar|ras|rgb|rmi|snd|svg|tar|tif|tiff|ttf|wav|woff|woff2|xbm|xpm|xwd|zip";
-	public static String[] excludeMIME = new String[] {"application/msword", "application/vnd.ms-project", "application/x-gzip", "application/x-tar", "application/zip", "audio/basic", "audio/mid", "audio/mpeg", "audio/x-aiff", "audio/x-mpegurl", "audio/x-pn-realaudio", "audio/x-wav", "image/bmp", "image/cis-cod", "image/gif", "image/ief", "image/jpeg", "image/png", "image/pipeg", "image/svg+xml", "image/tiff", "image/x-cmu-raster", "image/x-cmx", "image/x-icon", "image/x-portable-anymap", "image/x-portable-bitmap", "image/x-portable-graymap", "image/x-portable-pixmap", "image/x-rgb", "image/x-xbitmap", "image/x-xpixmap", "image/x-xwindowdump", "text/css", "video/mpeg", "video/mpeg", "application/font-woff"};
+	public static String[] engineArray = new String[] {"nfa", "dfa"};
+	
+	public static String excludeSuffix = "3g2|3gp|7z|aac|abw|aif|aifc|aiff|arc|au|avi|azw|bin|bmp|bz|bz2|cmx|cod|csh|css|csv|doc|docx|eot|epub|gif|gz|ico|ics|ief|jar|jfif|jpe|jpeg|jpg|m3u|mid|midi|mjs|mp2|mp3|mpa|mpe|mpeg|mpg|mpkg|mpp|mpv2|odp|ods|odt|oga|ogv|ogx|otf|pbm|pdf|pgm|png|pnm|ppm|ppt|pptx|ra|ram|rar|ras|rgb|rmi|rtf|snd|svg|swf|tar|tif|tiff|ttf|txt|vsd|wav|weba|webm|webp|woff|woff2|xbm|xls|xlsx|xpm|xul|xwd|zip|zip";
+	
+	public static String[] excludeMIME = new String[] {"application/epub+zip", "application/font-woff", "application/java-archive", "application/msword", "application/octet-stream", "application/ogg", "application/pdf", "application/rtf", "application/vnd.amazon.ebook", "application/vnd.apple.installer+xml", "application/vnd.mozilla.xul+xml", "application/vnd.ms-excel", "application/vnd.ms-fontobject", "application/vnd.ms-powerpoint", "application/vnd.ms-project", "application/vnd.oasis.opendocument.presentation", "application/vnd.oasis.opendocument.spreadsheet", "application/vnd.oasis.opendocument.text", "application/vnd.openxmlformats-officedocument.presentationml.presentation", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.visio", "application/x-7z-compressed", "application/x-abiword", "application/x-bzip", "application/x-bzip2", "application/x-csh", "application/x-freearc", "application/x-gzip", "application/x-rar-compressed", "application/x-shockwave-flash", "application/x-tar", "application/zip", "audio/3gpp", "audio/3gpp2", "audio/aac", "audio/basic", "audio/mid", "audio/midi audio/x-midi", "audio/mpeg", "audio/ogg", "audio/wav", "audio/webm", "audio/x-aiff", "audio/x-mpegurl", "audio/x-pn-realaudio", "audio/x-wav", "font/otf", "font/ttf", "font/woff", "font/woff2", "image/bmp", "image/cis-cod", "image/gif", "image/ief", "image/jpeg", "image/pipeg", "image/png", "image/svg+xml", "image/tiff", "image/vnd.microsoft.icon", "image/webp", "image/x-cmu-raster", "image/x-cmx", "image/x-icon", "image/x-portable-anymap", "image/x-portable-bitmap", "image/x-portable-graymap", "image/x-portable-pixmap", "image/x-rgb", "image/x-xbitmap", "image/x-xpixmap", "image/x-xwindowdump", "text/calendar", "text/css", "text/csv", "video/3gpp", "video/3gpp2", "video/mpeg", "video/ogg", "video/webm", "video/x-msvideo"};
+	
 	public static String outputTplString = "[%s]\n%s\n\n";
 }

burp/action/ExtractContent.java

@@ -1,7 +1,6 @@
 package burp.action;
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -9,6 +8,10 @@ import java.util.List;
 import org.json.JSONObject;
 
 import burp.file.ReadFile;
+import dk.brics.automaton.Automaton;
+import dk.brics.automaton.AutomatonMatcher;
+import dk.brics.automaton.RegExp;
+import dk.brics.automaton.RunAutomaton;
 import jregex.Matcher;
 import jregex.Pattern;
 
@@ -32,17 +35,31 @@ public class ExtractContent {
 				String scope = jsonObj1.getString("scope");
 				String action = jsonObj1.getString("action");
 				String color = jsonObj1.getString("color");
+				String engine = jsonObj1.getString("engine");
+				
 				List<String> result = new ArrayList<String>();
 				
 				if(isLoaded && (scope.equals(scopeString) || scope.equals("any")) && (action.equals(actionString) || action.equals("any"))) {
-					Pattern pattern = new Pattern(regex);
-					Matcher matcher = pattern.matcher(contentString);
-					while (matcher.find()) {
-						// 添加匹配数据至list
-						// 强制用户使用()包裹正则
-						result.add(matcher.group(1));
+					if (engine.equals("nfa")) {
+						Pattern pattern = new Pattern(regex);
+						Matcher matcher = pattern.matcher(contentString);
+						while (matcher.find()) {
+							// 添加匹配数据至list
+							// 强制用户使用()包裹正则
+							result.add(matcher.group(1));
+						}
+					} else {
+						RegExp regexpr = new RegExp(regex);
+						Automaton auto = regexpr.toAutomaton();
+				        RunAutomaton runAuto = new RunAutomaton(auto, true);
+				        AutomatonMatcher autoMatcher = runAuto.newMatcher(contentString);
+				        while (autoMatcher.find()) {
+							// 添加匹配数据至list
+							// 强制用户使用()包裹正则
+							result.add(autoMatcher.group());
+						}
 					}
-					
+
 					// 去除重复内容
 					HashSet tmpList = new HashSet(result);
 					result.clear();

burp/ui/FillTable.java

@@ -30,6 +30,7 @@ public class FillTable {
 			String color = jsonObj1.getString("color");
 			String scope = jsonObj1.getString("scope");
 			String action = jsonObj1.getString("action");
+			String engine = jsonObj1.getString("engine");
 			// 填充数据
 			Vector rules = new Vector();
 			rules.add(loaded);
@@ -38,6 +39,7 @@ public class FillTable {
 			rules.add(color);
 			rules.add(scope);
 			rules.add(action);
+			rules.add(engine);
 			dtm.addRow(rules);
 		}
 	}