# coding=utf-8 import fileinput import warnings import numpy as np import pandas as pd import matplotlib.pyplot as plt import os import tensorflow as tf import sys import re import nltk import sklearn import tensorflow.keras as keras import tensorflow.keras.preprocessing as keras_preprocessing from sklearn.preprocessing import StandardScaler import chardet import math from joblib import dump ''' os.environ["CUDA_VISIBLE_DEVICES"] = "0" config = tf.ConfigProto(allow_soft_placement=True) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5) config.gpu_options.allow_growth = True sess0 = tf.InteractiveSession(config=config) ''' g_word_dict = {} g_keras_token = None def os_listdir_ex(file_dir, find_name): # 祖传代码 result = [] for root, dirs, files in os.walk(file_dir): for file in files: if os.path.splitext(file)[1] == find_name: result.append(os.path.join(root, file)) # return result # 测试用 return result def get_file_length(pFile): # 得到文件长度,祖传代码 fsize = os.path.getsize(pFile) return int(fsize) def get_data_frame(): # 得到data frame # 得到webshell列表 webshell_files = os_listdir_ex(".\\webshell", '.php') # 得到正常文件列表 normal_files = os_listdir_ex(".\\noshell", '.php') label_webshell = [] label_normal = [] # 打上标注 for i in range(0, len(webshell_files)): label_webshell.append(1) for i in range(0, len(normal_files)): label_normal.append(0) # 合并起来 files_list = webshell_files + normal_files label_list = label_webshell + label_normal # 打乱数据,祖传代码 state = np.random.get_state() np.random.shuffle(files_list) # 训练集 np.random.set_state(state) np.random.shuffle(label_list) # 标签 data_list = {'label': label_list, 'file': files_list} return pd.DataFrame(data_list, columns=['label', 'file']) def flush_file(pFile): # 清洗php注释 file = open(pFile, 'r', encoding='gb18030', errors='ignore') read_string = file.read() file.close() m = re.compile(r'/\*.*?\*/', re.S) result = re.sub(m, '', read_string) m = re.compile(r'//.*') result = re.sub(m, '', result) m = re.compile(r'#.*') result = re.sub(m, '', result) return result # 得到文件熵 https://blog.csdn.net/jliang3/article/details/88359063 def get_file_entropy(pFile): clean_string = flush_file(pFile) text_list = {} _sum = 0 result = 0 for word_iter in clean_string: if word_iter != '\n' and word_iter != ' ': if word_iter not in text_list.keys(): text_list[word_iter] = 1 else: text_list[word_iter] = text_list[word_iter] + 1 for index in text_list.keys(): _sum = _sum + text_list[index] for index in text_list.keys(): result = result - float(text_list[index])/_sum * \ math.log(float(text_list[index])/_sum, 2) return result def vectorize_sequences(sequences, dimention=1337): # 创建一个大小为（25000，10000）的全零矩阵 results = np.zeros((len(sequences), dimention)) for i, sequence in enumerate(sequences): if i > dimention: break try: results[i, sequence] = 1. except: break return results def get_word_bag(pWordList): global g_word_dict global g_keras_token sequences_data = g_keras_token.texts_to_sequences(pWordList) word_bag = [] for index in range(0, len(sequences_data)): if len(sequences_data[index]) != 0: for zeus in range(0, len(sequences_data[index])): word_bag.append(sequences_data[index][zeus]) return word_bag def set_word_bag(pWordList): global g_word_dict global g_keras_token if g_keras_token == None: g_keras_token = keras.preprocessing.text.Tokenizer() # 初始化标注器 g_keras_token.fit_on_texts(pWordList) # 学习出文本的字典 g_word_dict.update(g_keras_token.word_index) def get_file_word(pFile): global g_word_dict english_punctuations = [',', '.', ':', ';', '?', '(', ')', '[', ']', '&', '!', '*', '@', '#', '$', '%', 'php', '<', '>', '\''] clean_string = flush_file(pFile) word_list = nltk.word_tokenize(clean_string) # 过滤掉不干净的 word_list = [ word_iter for word_iter in word_list if word_iter not in english_punctuations] # anti-paste return word_list def get_file_word_bag_non_use(pFile): global g_word_dict english_punctuations = [',', '.', ':', ';', '?', '(', ')', '[', ']', '&', '!', '*', '@', '#', '$', '%', 'php', '<', '>', '\''] clean_string = flush_file(pFile) word_list = nltk.word_tokenize(clean_string) # 过滤掉不干净的 word_list = [ word_iter for word_iter in word_list if word_iter not in english_punctuations] keras_token = keras.preprocessing.text.Tokenizer() # 初始化标注器 keras_token.fit_on_texts(word_list) # 学习出文本的字典 g_word_dict.update(keras_token.word_index) # 通过texts_to_sequences 这个dict可以将每个string的每个词转成数字 sequences_data = keras_token.texts_to_sequences(word_list) # 将每条文本的长度设置一个固定值, ps 超过1337个字符的"单词"不用说肯定是某个骇客想把大马变免杀马 # word_bag = keras_preprocessing.sequence.pad_sequences(sequences_data, maxlen=1337, dtype='int16') word_bag = [] for index in range(0, len(sequences_data)): if len(sequences_data[index]) != 0: for zeus in range(0, len(sequences_data[index])): word_bag.append(sequences_data[index][zeus]) ''' word_bag = vectorize_sequences(word_bag) finished_bag = [] for item in range(0, len(word_bag)): if len(word_bag[item]) != 0: for zeus in range(0, len(word_bag[item])): finished_bag.append(word_bag[item][zeus]) print(len(finished_bag)) ''' ''' tmp_fill = np.zeros((1337,), dtype=np.float) max_get_len = 1337 if len(word_bag) < max_get_len: while True: word_bag = np.insert(word_bag, len(word_bag) - 1, tmp_fill, axis=0) if len(word_bag) >= max_get_len: break elif len(word_bag) > max_get_len: np.delete(word_bag, np.s_[max_get_len:]) ''' return word_bag def build_network(): global g_word_dict # y = label # 进来的file length_scaled entropy_scaled word_bag # 第一网络是一个TextCNN 词嵌入-卷积池化*3-拼接-全连接-dropout-全连接 input_1 = keras.layers.Input( shape=(1337,), dtype='int16', name='word_bag') # 词嵌入（使用预训练的词向量） embed = keras.layers.Embedding( len(g_word_dict) + 1, 300, input_length=1337)(input_1) # 词窗大小分别为3,4,5 cnn1 = keras.layers.Conv1D( 256, 3, padding='same', strides=1, activation='relu')(embed) cnn1 = keras.layers.MaxPooling1D(pool_size=48)(cnn1) cnn2 = keras.layers.Conv1D( 256, 4, padding='same', strides=1, activation='relu')(embed) cnn2 = keras.layers.MaxPooling1D(pool_size=47)(cnn2) cnn3 = keras.layers.Conv1D( 256, 5, padding='same', strides=1, activation='relu')(embed) cnn3 = keras.layers.MaxPooling1D(pool_size=46)(cnn3) # 合并三个模型的输出向量 cnn = keras.layers.concatenate([cnn1, cnn2, cnn3], axis=1) flat = keras.layers.Flatten()(cnn) drop = keras.layers.Dropout(0.2)(flat) model_1_output = keras.layers.Dense( 1, activation='sigmoid', name='TextCNNoutPut')(drop) # 第一层好了 # model_1 = keras.Model(inputs=input_1, outputs=model_1_output) # 第二层 input_2 = keras.layers.Input( shape=(2,), dtype='float32', name='length_entropy') model_2 = keras.layers.Dense( 128, input_shape=(2,), activation='relu')(input_2) model_2 = keras.layers.Dropout(0.4)(model_2) model_2 = keras.layers.Dense(64, activation='relu')(model_2) model_2 = keras.layers.Dropout(0.2)(model_2) model_2 = keras.layers.Dense(32, activation='relu')(model_2) model_2_output = keras.layers.Dense( 1, activation='sigmoid', name='LengthEntropyOutPut')(model_2) # 第二层好了 # 此时，我们将辅助输入数据与TextCNN层的输出连接起来,输入到模型中 model_combined = keras.layers.concatenate([model_2_output, model_1_output]) model_end = keras.layers.Dense(64, activation='relu')(model_combined) model_end = keras.layers.Dense( 1, activation='sigmoid', name='main_output')(model_end) # 定义这个具有两个输入和输出的模型 model_end = keras.Model(inputs=[input_2, input_1], outputs=model_end) model_end.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) ''' # 第二层就是自己写的128 - drop 0.4 - 64 - drop 0.2 - 32 简单地分类,2输入 length_scaled entropy_scaled model_2 = keras.Sequential() model_2.add(keras.layers.Dense( 128, input_shape=(2,), activation='relu')) model_2.add(keras.layers.Dropout(0.4)) model_2.add(keras.layers.Dense( 64, activation='relu')) model_2.add(keras.layers.Dropout(0.2)) model_2.add(keras.layers.Dense( 32, activation='relu')) # 第二网络好了 model_2.add(keras.layers.Dense(1, activation='sigmoid')) # 融在一起 # model_1.summary() model_combined = keras.layers.concatenate([model_2.output, model_1.output]) # 融在一起 model_end = keras.layers.Dense( 2, activation="relu")(model_combined) model_end = keras.layers.Dense(1, activation="sigmoid")(model_end) model_end = keras.Model( inputs=[model_2.input, model_1.input], outputs=model_end) model_end.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy', 'loss']) # model_end = keras.layers.Dense(2, activation="relu")(model_combined) # model_end = keras.layers.Dense(1, activation="sigmod")(model_end) # model_end = keras.Model(inputs=[x.input, y.input], outputs=z) ''' return model_end # get_functions("C:\\Users\\Administrator\\Desktop\\webshell检测\\webshell\\一句话\\一句话.php") data_frame = [] if os.path.exists("save.csv") == False: data_frame = get_data_frame() print(data_frame.head(5)) data_frame['length'] = data_frame['file'].map( lambda file_name: get_file_length(file_name)).astype(int) data_frame['entropy'] = data_frame['file'].map( lambda file_name: get_file_entropy(file_name)).astype(float) # 归一化这两个东西 scaler_length = StandardScaler() scaler_entropy = StandardScaler() data_frame['length_scaled'] = scaler_length.fit_transform( data_frame['length'].values.reshape(-1, 1), scaler_length.fit(data_frame['length'].values.reshape(-1, 1))) data_frame['entropy_scaled'] = scaler_entropy.fit_transform( data_frame['entropy'].values.reshape(-1, 1), scaler_entropy.fit(data_frame['entropy'].values.reshape(-1, 1))) # 导入词袋 data_frame['word_bag'] = data_frame['file'].map( lambda file_name: get_file_word(file_name)) set_word_bag(data_frame['word_bag']) data_frame['word_bag'] = data_frame['word_bag'].map( lambda text_params: get_word_bag(text_params)) data_frame.to_csv("save.csv") dump(scaler_length, 'scaler_length.joblib') dump(scaler_entropy, 'scaler_entropy.joblib') else: data_frame = pd.read_csv("save.csv", header=0) print(data_frame.head(5)) skip_Data_num = 2610 data_train_pre = data_frame.filter( items=['length_scaled', 'entropy_scaled']) data_train_y = tf.constant(data_frame.filter( items=['label'])[:skip_Data_num]) data_train_x_1 = tf.constant(data_train_pre[:skip_Data_num]) data_train_x_2 = tf.constant( vectorize_sequences(data_frame['word_bag'].values[:skip_Data_num])) # 现在这个是一个 (batch_size,(1337个单词[1337个hot code])) network_model = build_network() network_model.summary() history = network_model.fit( x=[data_train_x_1, data_train_x_2], y=data_train_y, epochs=100) network_model.save('huoji.h5')