import tensorflow as tf
print(tf.__version__)
import torch as th

print(th.version)
import keras

print(keras.version)

import tensorflow as tf

print(tf.version)

import os

os.environ["CUDA_DEVICES_ORDER"] = "PCI_BUS_IS"

os.environ["CUDA_VISIBLE_DEVICES"] = "0"
#sess = tf.Session()

sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))

a = tf.constant(1)

b = tf.constant(2)

print(sess.run(a+b))
from tensorflow.python.client import device_lib

print(device_lib.list_local_devices())

import tensorflow as tf

with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)) as sess:

sess.run(init)

with tf.device('/device:GPU:0'):

for _ in range(3):

print(sess.run(state))

from sklearn.datasets import load_iris

import numpy as np

from sklearn.linear_model import LinearRegression

import matplotlib.pyplot as plt

#获取花瓣的长和宽

hua = load_iris()

x = [n[0] for n in hua.data]

y = [n[1] for n in hua.data]
x = np.array(x).reshape(len(x),1)

y = np.array(y).reshape(len(y),1)
#线性回归分析

clf = LinearRegression()

clf.fit(x,y)

pre = clf.predict(x)
#可视化画图

plt.scatter(x,y,s=100)

plt.plot(x,pre,"r-",linewidth=4)

for idx, m in enumerate(x):

plt.plot([m,m],[y[idx],pre[idx]], 'g-')

plt.show()

# coding: utf-8

import pandas as pd

import jieba

import time

import csv

from sklearn.feature_extraction.text import TfidfVectorizer

#----------------------------------自定义函数 N-Gram处理--------------------------------
tokenizer function, this will make 3 grams of each query
www.foo.com/1 转换为 ['www','ww.','w.f','.fo','foo','oo.','o.c','.co','com','om/','m/1']
def get_ngrams(query):

tempQuery = str(query)

ngrams = []

for i in range(0, len(tempQuery)-3+1):

ngrams.append(tempQuery[i:i+3])

return ngrams
#----------------------------------主函数 读取文件及预处理-------------------------------

if name == 'main':

# 使用csv.DictReader读取文件中的信息

file = "all_data_url_random.csv"

with open(file, "r", encoding="UTF-8") as f:

reader = csv.DictReader(f)

labels = []

contents = []

for row in reader:

# 数据元素获取

labels.append(row['label'])

contents.append(row['content'])

print(labels[:10])

print(contents[:10])
#文件写入
#数据划分 前10000-训练集 中间5000-测试集 后5000-验证集
ctrain = open("all_data_url_random_fenci_train.csv", "a+", newline='', encoding='gb18030')
writer1 = csv.writer(ctrain)
writer1.writerow(["label","fenci"])

ctest = open("all_data_url_random_fenci_test.csv", "a+", newline='', encoding='gb18030')
writer2 = csv.writer(ctest)
writer2.writerow(["label","fenci"])

cval = open("all_data_url_random_fenci_val.csv", "a+", newline='', encoding='gb18030')
writer3 = csv.writer(cval)
writer3.writerow(["label","fenci"])

n = 0
while n < len(contents):
    res = get_ngrams(contents[n])
    #print(res)
    final = ' '.join(res)
    tlist = []
    tlist.append(labels[n])
    tlist.append(final)
    if n<10000:
        writer1.writerow(tlist)  #训练集
    elif n>=10000 and n<15000:
        writer2.writerow(tlist)  #测试集
    elif n>=15000:
        writer3.writerow(tlist)  #验证集
    n = n + 1

#文件关闭
ctrain.close()
ctest.close()
cval.close()</code></pre></div></div><p>输出结果如下图所示，包括训练集、测试集和验证集。</p><figure class=""><div class="rno-markdown-img-url" style="text-align:center"><div class="rno-markdown-img-url-inner" style="width:auto"><div style="width:100%"><img src="https://cdn.static.attains.cn/app/developer-bbs/upload/1722944790541824860.png" /></div></div></div></figure><figure class=""><hr/></figure><h3 id="c18pt" name="2.LSTM%E6%9E%84%E5%BB%BA">2.LSTM构建</h3><p>该部分包括两个Python文件，具体如下：</p><ul class="ul-level-0"><li>LSTM_data.py：构建LSTM模型，实现恶意请求分类功能</li><li>load_pj.py：算法评价模型，自定义计算分类的Precision、Recall和F-measure</li></ul><p>LSTM_data.py

具体任务包括六个步骤，代码中包括详细的注释，如下：
第一步，读取数据
第二步，进行OneHotEncoder()编码，可以采用TF-IDF、Word2Vec等方法
第三步，使用Tokenizer对词组进行编码，将数据转换为固定长度的词序列
第四步，建立LSTM模型，模型如下图所示
第五步，设置flag开关进行模型训练和模型预测，模型评估调用 load_pj.py 实现，通弄湿绘制热力图
第六步，验证算法进行验证集的测试
代码语言：javascript
复制
# coding=utf-8

By：Eastmount CSDN 2020-11-15
import pickle

import pandas as pd

import numpy as np

import matplotlib.pyplot as plt

import seaborn as sns

from sklearn import metrics

from sklearn.preprocessing import LabelEncoder,OneHotEncoder

from keras.models import Model

from keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding

from keras.optimizers import RMSprop

from keras.preprocessing.text import Tokenizer

from keras.preprocessing import sequence

from keras.callbacks import EarlyStopping

from keras.models import load_model

from load_pj import classification_pj

import time
start = time.clock()
#---------------------------------------第一步 数据读取------------------------------------

#读取测数据集

train_df = pd.read_csv("all_data_url_random_fenci_train.csv")

val_df = pd.read_csv("all_data_url_random_fenci_val.csv")

test_df = pd.read_csv("all_data_url_random_fenci_test.csv")

print(train_df.head())
#解决中文显示问题

plt.rcParams['font.sans-serif'] = ['KaiTi'] # 指定默认字体 SimHei黑体

plt.rcParams['axes.unicode_minus'] = False # 解决保存图像是负号'
#---------------------------------第二步 OneHotEncoder()编码---------------------------------

#对数据集的标签数据进行编码

train_y = train_df.label

print("Label:")

print(train_y[:10])
val_y = val_df.label

test_y = test_df.label

le = LabelEncoder()

train_y = le.fit_transform(train_y).reshape(-1,1)

print("LabelEncoder")

print(train_y[:10])

print(len(train_y))
val_y = le.transform(val_y).reshape(-1,1)

test_y = le.transform(test_y).reshape(-1,1)
对数据集的标签数据进行one-hot编码
ohe = OneHotEncoder()

train_y = ohe.fit_transform(train_y).toarray()

val_y = ohe.transform(val_y).toarray()

test_y = ohe.transform(test_y).toarray()

print("OneHotEncoder:")

print(train_y[:10])
#-------------------------------第三步 使用Tokenizer对词组进行编码-------------------------------

#使用Tokenizer对词组进行编码

#当我们创建了一个Tokenizer对象后，使用该对象的fit_on_texts()函数，以空格去识别每个词

#可以将输入的文本中的每个词编号，编号是根据词频的，词频越大，编号越小

max_words = 5000

max_len = 600

tok = Tokenizer(num_words=max_words)  #使用的最大词语数为5000

tok.fit_on_texts(train_df.fenci)

print(tok)
#保存训练好的Tokenizer和导入

with open('tok.pickle', 'wb') as handle:

pickle.dump(tok, handle, protocol=pickle.HIGHEST_PROTOCOL)
loading
with open('tok.pickle', 'rb') as handle:

tok = pickle.load(handle)
#使用word_index属性可以看到每次词对应的编码

#使用word_counts属性可以看到每个词对应的频数

for ii,iterm in enumerate(tok.word_index.items()):

if ii < 10:

print(iterm)

else:

break

print("===================")

for ii,iterm in enumerate(tok.word_counts.items()):

if ii < 10:

print(iterm)

else:

break
#使用tok.texts_to_sequences()将数据转化为序列

#使用sequence.pad_sequences()将每个序列调整为相同的长度

#对每个词编码之后，每句语料中的每个词就可以用对应的编码表示，即每条语料可以转变成一个向量了

train_seq = tok.texts_to_sequences(train_df.fenci)

val_seq = tok.texts_to_sequences(val_df.fenci)

test_seq = tok.texts_to_sequences(test_df.fenci)
#将每个序列调整为相同的长度

train_seq_mat = sequence.pad_sequences(train_seq,maxlen=max_len)

val_seq_mat = sequence.pad_sequences(val_seq,maxlen=max_len)

test_seq_mat = sequence.pad_sequences(test_seq,maxlen=max_len)
print(train_seq_mat.shape)  #(10000, 600)

print(val_seq_mat.shape)    #(5000, 600)

print(test_seq_mat.shape)   #(5000, 600)

print(train_seq_mat[:2])
#-------------------------------第四步 建立LSTM模型并训练-------------------------------
定义LSTM模型
inputs = Input(name='inputs',shape=[max_len])
Embedding(词汇表大小,batch大小,每个新闻的词长)
layer = Embedding(max_words+1, 128, input_length=max_len)(inputs)

layer = LSTM(128)(layer)

layer = Dense(128, activation="relu", name="FC1")(layer)

layer = Dropout(0.3)(layer)

layer = Dense(2, activation="softmax", name="FC2")(layer)

model = Model(inputs=inputs, outputs=layer)

model.summary()

model.compile(loss="categorical_crossentropy",

optimizer=RMSprop(),

metrics=["accuracy"])
增加判断 防止再次训练
flag = "train"

if flag == "train":

print("模型训练")

#模型训练

model_fit = model.fit(train_seq_mat, train_y, batch_size=128, epochs=10,

validation_data=(val_seq_mat,val_y),

callbacks=[EarlyStopping(monitor='val_loss',min_delta=0.0001)]   #当val-loss不再提升时停止训练

)
#保存模型
model.save(&#39;my_model.h5&#39;)  
del model  # deletes the existing model

#计算时间
elapsed = (time.clock() - start)
print(&#34;Time used:&#34;, elapsed)

else:

print("模型预测")

# 导入已经训练好的模型

model = load_model('my_model.h5')
#--------------------------------------第五步 预测及评估--------------------------------
#对测试集进行预测
test_pre = model.predict(test_seq_mat)

#评价预测效果，计算混淆矩阵 参数顺序
confm = metrics.confusion_matrix(np.argmax(test_y,axis=1),np.argmax(test_pre,axis=1))
print(confm)
#混淆矩阵可视化
Labname = [&#39;正常&#39;, &#39;异常&#39;]

print(metrics.classification_report(np.argmax(test_y,axis=1),np.argmax(test_pre,axis=1)))
classification_pj(np.argmax(test_pre,axis=1),np.argmax(test_y,axis=1))

plt.figure(figsize=(8,8))
sns.heatmap(confm.T, square=True, annot=True,
            fmt=&#39;d&#39;, cbar=False, linewidths=.6,
            cmap=&#34;YlGnBu&#34;)
plt.xlabel(&#39;True label&#39;,size = 14)
plt.ylabel(&#39;Predicted label&#39;, size = 14)
plt.xticks(np.arange(2)+0.8, Labname, size = 12)
plt.yticks(np.arange(2)+0.4, Labname, size = 12)
plt.show()

#--------------------------------------第六步 验证算法--------------------------------
#使用tok对验证数据集重新预处理，并使用训练好的模型进行预测
val_seq = tok.texts_to_sequences(val_df.fenci)

#将每个序列调整为相同的长度
val_seq_mat = sequence.pad_sequences(val_seq,maxlen=max_len)

#对验证集进行预测
val_pre = model.predict(val_seq_mat)
print(metrics.classification_report(np.argmax(val_y,axis=1),np.argmax(val_pre,axis=1)))
classification_pj(np.argmax(val_pre,axis=1),np.argmax(val_y,axis=1))

#计算时间
elapsed = (time.clock() - start)
print(&#34;Time used:&#34;, elapsed)</code></pre></div></div><p>输出的部分结果如下图所示：</p><div class="rno-markdown-code"><div class="rno-markdown-code-toolbar"><div class="rno-markdown-code-toolbar-info"><div class="rno-markdown-code-toolbar-item is-type"><span class="is-m-hidden">代码语言：</span>javascript</div></div><div class="rno-markdown-code-toolbar-opt"><div class="rno-markdown-code-toolbar-copy"><i class="icon-copy"></i><span class="is-m-hidden">复制</span></div></div></div><div class="developer-code-block"><pre class="prism-token token line-numbers language-javascript"><code class="language-javascript" style="margin-left:0"> label                                              fenci

0    异常                            /.. ../ ./. /.s .sl sln

1    异常  /fc fc9 c9t 9t5 t54 54l 4l7 l7. 7.c .cf cfm fm...

2    正常  /10 108 085 856 569 69_ 9_m mi mis iss ssi si...

3    异常  /sc scr cri rip ipt pts ts/ s/f /fo for oru ru...

4    正常            /ro row ow2 w2 2_j _jo joi oin int nt/
LabelEncoder

[[0]

[0]

[1]

[0]

[1]

[1]

[1]

[1]

[1]

[0]]

10000
OneHotEncoder:

[[1. 0.]

[1. 0.]

[0. 1.]

[1. 0.]

[0. 1.]

[0. 1.]

[0. 1.]

[0. 1.]

[0. 1.]

[1. 0.]]
<keras_preprocessing.text.Tokenizer object at 0x00000208C5644C50>

('s', 7660)

('sl', 362)

('sln', 1)

('fc', 53)

('fc9', 4)

('c9t', 2)

('9t5', 2)

('t54', 3)

('54l', 2)

('4l7', 8)

(10000, 600)

(5000, 600)

(5000, 600)

[[  0   0   0 ...   0   4 199]

[  0   0   0 ...  17  19  24]]
训练过程如下所示：
代码语言：javascript
复制
Train on 10000 samples, validate on 5000 samples

Epoch 1/10

10000/10000 [==============================] - 250s 25ms/step - loss: 0.2590 - accuracy: 0.9102 - val_loss: 0.1295 - val_accuracy: 0.9626

Epoch 2/10

10000/10000 [==============================] - 248s 25ms/step - loss: 0.1783 - accuracy: 0.9616 - val_loss: 0.1642 - val_accuracy: 0.9366

Time used: 502.5310856
3.实验评估
当实验训练结束后，将flag变量设置为“test”进行测试，评估代码如下：
代码语言：javascript
复制
# -*- coding: utf-8 -*-

"""

author: Eastmount CSDN 2020-11-15

"""

import os

#评价指标 参数顺序

def classification_pj(pre, y_test):

# 正确率 Precision = 正确识别的个体总数 /识别出的个体总数

# 召回率 Recall = 正确识别的个体总数 /  测试集中存在的个体总数

# F值 F-measure = 正确率 * 召回率 * 2 / (正确率 + 召回率)
YC_A, YC_B = 0,0  #预测 bad good
ZQ_A, ZQ_B = 0,0  #正确
CZ_A, CZ_B = 0,0  #存在

#0-good 1-bad 同时计算防止类标变化
i = 0
while i&lt;len(pre):
    z = int(y_test[i])   #真实 
    y = int(pre[i])      #预测

    if z==0:
        CZ_A += 1
    elif z==1:
        CZ_B += 1
        
    if y==0:
        YC_A += 1
    elif y==1:
        YC_B += 1
        
    if z==y and z==0 and y==0:
        ZQ_A += 1
    elif z==y and z==1 and y==1:
        ZQ_B += 1
    i = i + 1

# 结果输出
print(YC_A, YC_B, ZQ_A, ZQ_B,CZ_A, CZ_B)
P_A = ZQ_A * 1.0 / (YC_A + 0.1)
P_B = ZQ_B * 1.0 / (YC_B + 0.1)
print(&#34;Precision 0:{:.4f}&#34;.format(P_A))
print(&#34;Precision 1:{:.4f}&#34;.format(P_B))
print(&#34;Avg_precision:{:.4f}&#34;.format((P_A + P_B)/2))

R_A = ZQ_A * 1.0 / (CZ_A + 0.1)
R_B = ZQ_B * 1.0 / (CZ_B + 0.1)
print(&#34;Recall 0:{:.4f}&#34;.format(R_A))
print(&#34;Recall 1:{:.4f}&#34;.format(R_B))
print(&#34;Avg_recall:{:.4f}&#34;.format((R_A + R_B)/2))

F_A = 2 * P_A * R_A / (P_A + R_A)
F_B = 2 * P_B * R_B / (P_B + R_B)
print(&#34;F-measure 0:{:.4f}&#34;.format(F_A))
print(&#34;F-measure 1:{:.4f}&#34;.format(F_B))
print(&#34;Avg_fmeasure:{:.4f}&#34;.format((F_A + F_B)/2))
print(&#34;&#34;)</code></pre></div></div><p>输出结果如下图所示：</p><ul class="ul-level-0"><li>F值为0.9488</li><li>P值为0.9533</li><li>R值为0.9450</li></ul><div class="rno-markdown-code"><div class="rno-markdown-code-toolbar"><div class="rno-markdown-code-toolbar-info"><div class="rno-markdown-code-toolbar-item is-type"><span class="is-m-hidden">代码语言：</span>javascript</div></div><div class="rno-markdown-code-toolbar-opt"><div class="rno-markdown-code-toolbar-copy"><i class="icon-copy"></i><span class="is-m-hidden">复制</span></div></div></div><div class="developer-code-block"><pre class="prism-token token line-numbers language-javascript"><code class="language-javascript" style="margin-left:0">模型预测

[[1758  164]

[  76 3002]]

precision    recall  f1-score   support
       0       0.96      0.91      0.94      1922
       1       0.95      0.98      0.96      3078

accuracy                           0.95      5000

macro avg       0.95      0.94      0.95      5000

weighted avg       0.95      0.95      0.95      5000
1834 3166 1758 3002 1922 3078

Precision 0:0.9585

Precision 1:0.9482

Avg_precision:0.9533

Recall 0:0.9146

Recall 1:0.9753

Avg_recall:0.9450

F-measure 0:0.9361

F-measure 1:0.9615

Avg_fmeasure:0.9488
当然你还可以和机器学习算法进行对比喔！前一篇文章作者不是详细讲解了机器学习的对比实验吗？比如决策树的运行结果如下图所示：
F值为0.93
注意：大家可以直接在易学智能平台的PyCharm运行代码，也可以在自己搭建的环境运行。
如果是GPU环境需要增加一些核心代码：
ailed to create cublas handle: CUBLAS_STATUS_ALLOC_FAILED
failed to allocate 9.90G (10630043904 bytes) from device: CUDA_ERROR_OUT_OF_MEMORY: out of memory
增加下面代码：
代码语言：javascript
复制
import os

os.environ["CUDA_DEVICES_ORDER"] = "PCI_BUS_IS"

os.environ["CUDA_VISIBLE_DEVICES"] = "0"

#指定了每个GPU进程中使用显存的上限,0.9表示可以使用GPU 90%的资源进行训练

gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)

sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
"""

#TensorFlow2.0

gpus = tf.config.experimental.list_physical_devices('GPU')

for gpu in gpus:

tf.config.experimental.set_memory_growth(gpu,True)

"""
五.总结
写到这里，这篇文章就结束了，下一篇文章比较硬核，直接撰写代码详细对比CNN、LSTM、BiLSTM和BiLSTM+Attention文本分类实验。希望对您有所帮助，同时文章中不足或错误的地方，欢迎读者提出。这些实验都是我在做论文研究或项目评价常见的一些问题，希望读者带着这些问题，结合自己的需求进行深入的思考，更希望大家能学以致用。最后如果文章对您有帮助，请点赞、评论、收藏，这将是我分享最大的动力。github下载代码，记得关注点赞喔！
https://github.com/eastmountyxz/AI-for-Keras
最后，作为人工智能的菜鸟，我希望自己能不断进步并深入，后续将它应用于图像识别、网络安全、对抗样本等领域，指导大家撰写简单的学术论文，一起加油！感谢这些年遇到很多以前进步的博友，共勉~
2020年新开的“娜璋AI安全之家”，主要围绕Python大数据分析、网络空间安全、人工智能、Web渗透及攻防技术进行讲解，同时分享CCF、SCI、南核北核论文的算法实现。娜璋之家会更加系统，并重构作者的所有文章，从零讲解Python和安全，写了近十多年文章，真心想把自己所学所感所做分享出来，还请各位多多指教，真诚邀请您的关注！谢谢。
(By:Eastmount 2024-03-12 夜于火星)