第5讲讲解了keras制作mnist数据集的流程，进一步的，有时候我们需要构建自己的数据集。
以flower分类为例，见参考3（这里直接用别人的数据，也可以自己整理，只是比较耗时间，脚本也做了改写，方便理解）
step1：下载flower数据集
数据下载地址见参考3，下载完成后得到5个文件夹，分别存放5种不同花的jpg图像：
flower_photos
# tree -L 1
.
├── LICENSE.txt
├── daisy
# 雏菊，编号0
├── dandelion
# 蒲公英，编号1
├── roses
# 玫瑰，编号2
├── sunflowers
# 向日葵，编号3
└── tulips
# 郁金香，编号4step2：制作图像-标签字典
我们需要给图像打上标签，比如0代表daisy，1代表dandelion，等，神经网络训练完成后，输入一个上述5种花的一种，经过模型处理后，得到标签[0-4]，这个标签表示花的种类。
# create_image2label.py
import os,
glob,
json,
random
def create_image2label(
root,
jsonname):
'''
root:images文件夹路径
jsonname：写入的json文件名
'''
name2label = {}
# step1: 将flower_photos文件夹依次编号
for name in sorted(
os.
listdir(
root)):
# 如果flower_photos/name不是文件夹，跳过
if not os.
path.
isdir(
os.
path.
join(
root,
name)):
continue
else:
# 依次编号
name2label[
name] =
len(
name2label.
keys())
# 这样得到 name2label = {'daisy': 0, 'dandelion': 1, 'roses': 2, 'sunflowers': 3, 'tulips': 4}
# step2: 将image-label对应关系写入到json中
if not os.
path.
exists(
os.
path.
join(
root,
jsonname)):
images = []
for name in name2label.
keys():
# 返回图片路径：flower_photos/xxx/xxx, 支持jpg/png等格式
images +=
glob.
glob(
os.
path.
join(
root,
name,
"*.*g"))
# 图片按类别写入列表，需要打乱顺序，原位打乱
random.
shuffle(
images)
# 用来保存图片路径-label对应关系
path_label = {}
for img in images:
# 将路径按/或\分离[flower_photos, daisy, **.jpg]取出类别名
name =
img.
split(
os.
sep)[
-2]
# 根据字典key，对应value找出对应标签
label =
name2label[
name]
path_label[
img] =
label
# 将路径,标签信息写入json文件
with open(
os.
path.
join(
root,
jsonname),
'w',
newline=
"")
as js:
json.
dump(
path_label,
js)
if __name__ ==
'__main__':
cwd =
'./flower_photos'
jsonname =
'images.json'
create_image2label(
cwd,
jsonname)
在当前路径下得到images.json文件，里面记录了图片路径与标签，如下（截取部分）。
{
"./flower_photos/daisy/7133935763_82b17c8e1b_n.jpg":
0,
"./flower_photos/sunflowers/5020805135_1219d7523d.jpg":
3,
"./flower_photos/roses/10090824183_d02c613f10_m.jpg":
2,
"./flower_photos/daisy/9346508462_f0af3163f4.jpg":
0,
"./flower_photos/sunflowers/9655029591_7a77f87500.jpg":
3,
"./flower_photos/sunflowers/22255608949_172d7c8d22_m.jpg":
3,
"./flower_photos/tulips/113291410_1bdc718ed8_n.jpg":
4,
...
}
step3：load数据以及数据的预处理
import os,
glob,
json,
random
import tensorflow as tf
import numpy as np
def load_json(
root,
jsonname):
images,
labels = [], []
with open(
os.
path.
join(
root,
jsonname))
as js:
jscontent =
json.
load(
js)
for key,
value in jscontent.
items():
label =
int(
value)
images.
append(
key)
labels.
append(
label)
assert len(
images) ==
len(
labels)
return images,
labels
def load_data(
root,
mode=
'train'):
jsonname =
'images.json'
images,
labels =
load_json(
root,
jsonname)
# 训练集
if mode ==
'train':
# 60%
images =
images[:
int(
0.6 * len(
images))]
labels =
labels[:
int(
0.6 * len(
labels))]
# 验证集
elif mode ==
'val':
# 20% = 60%->80%
images =
images[
int(
0.6 * len(
images)):
int(
0.8 * len(
images))]
labels =
labels[
int(
0.6 * len(
labels)):
int(
0.8 * len(
labels))]
# 测试集
else:
# 20% = 80%->100%
images =
images[
int(
0.8 * len(
images)):]
labels =
labels[
int(
0.8 * len(
labels)):]
return images,
labels
def preprocess(
x,
y):
x =
tf.
io.
read_file(
x)
x =
tf.
image.
decode_jpeg(
x,
channels=
3)
x =
tf.
image.
resize(
x, [
244,
244])
x =
tf.
cast(
x,
dtype=
tf.
float32)
/ 255.0
y =
tf.
convert_to_tensor(
y)
y =
tf.
one_hot(
y,
depth=
5)
return x,
y
if __name__ ==
'__main__':
cwd =
'./flower_photos'
batchsz =
32
# 创建训练集Datset对象
images,
labels =
load_data(
cwd,
mode=
'train')
db_train =
tf.
data.
Dataset.
from_tensor_slices((
images,
labels))
db_train =
db_train.
shuffle(
1000).
map(
preprocess).
batch(
batchsz)
# 创建验证集Datset对象
images2,
labels2 =
load_data(
cwd,
mode=
'val')
db_val =
tf.
data.
Dataset.
from_tensor_slices((
images2,
labels2))
db_val =
db_val.
map(
preprocess).
batch(
batchsz)
# 创建测试集Datset对象
images3,
labels3 =
load_data(
cwd,
mode=
'test')
db_test =
tf.
data.
Dataset.
from_tensor_slices((
images3,
labels3))
db_test =
db_test.
map(
preprocess).
batch(
batchsz)
这样得到训练数据集db_train，验证数据集db_val，测试数据集db_test
step4：训练验证
# train.py
import os
import tensorflow as tf
import numpy as np
from tensorflow import keras
from tensorflow.
keras import layers,
optimizers,
losses
from tensorflow.
keras.
callbacks import EarlyStopping
from load_data import load_data,
preprocess
if __name__ ==
'__main__':
cwd =
'./flower_photos'
batchsz =
32
# 创建训练集Datset对象
images,
labels =
load_data(
cwd,
mode=
'train')
db_train =
tf.
data.
Dataset.
from_tensor_slices((
images,
labels))
db_train =
db_train.
shuffle(
1000).
map(
preprocess).
batch(
batchsz)
# 创建验证集Datset对象
images2,
labels2 =
load_data(
cwd,
mode=
'val')
db_val =
tf.
data.
Dataset.
from_tensor_slices((
images2,
labels2))
db_val =
db_val.
map(
preprocess).
batch(
batchsz)
# 创建测试集Datset对象
images3,
labels3 =
load_data(
cwd,
mode=
'test')
db_test =
tf.
data.
Dataset.
from_tensor_slices((
images3,
labels3))
db_test =
db_test.
map(
preprocess).
batch(
batchsz)
#创建网络模型
net =
keras.
applications.
VGG16(
weights=
'imagenet',
include_top=
False,
pooling=
'max')
net.
trainable =
False
newnet =
keras.
Sequential([
net,
layers.
Dense(
128),
layers.
Dense(
64),
layers.
Dense(
5)
])
newnet.
build(
input_shape=(
4,
224,
224,
3))
newnet.
summary()
early_stopping =
EarlyStopping(
monitor=
'val_accuracy',
min_delta=
0.001,
patience=
5
)
newnet.
compile(
optimizer=
optimizers.
Adam(
lr=
1e-3),
loss=
losses.
CategoricalCrossentropy(
from_logits=
True),
metrics=[
'accuracy'])
# 为了节省时间，epochs取5，为了提高识别率，这里可以增大epochs
newnet.
fit(
db_train,
validation_data=
db_val,
validation_freq=
1,
epochs=
5,
callbacks=[
early_stopping])
# 用测试集测试模型准确率
newnet.
evaluate(
db_test)
测试log如下：
Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
vgg16 (Functional) (None, 512) 14714688
dense (Dense) (None, 128) 65664
dense_1 (Dense) (None, 64) 8256
dense_2 (Dense) (None, 5) 325
=================================================================
Total params: 14,788,933
Trainable params: 74,245
Non-trainable params: 14,714,688
_________________________________________________________________
/usr/local/lib/python3.8/dist-packages/keras/optimizers/optimizer_v2/adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
super(Adam, self).__init__(name, **kwargs)
Epoch 1/5
69/69 [==============================] - 181s 3s/step - loss: 1.0423 - accuracy: 0.6190 - val_loss: 0.6405 - val_accuracy: 0.7684
Epoch 2/5
69/69 [==============================] - 188s 3s/step - loss: 0.5819 - accuracy: 0.7929 - val_loss: 0.5232 - val_accuracy: 0.8147
Epoch 3/5
69/69 [==============================] - 169s 2s/step - loss: 0.4582 - accuracy: 0.8361 - val_loss: 0.5095 - val_accuracy: 0.8256
Epoch 4/5
69/69 [==============================] - 187s 3s/step - loss: 0.3972 - accuracy: 0.8615 - val_loss: 0.5235 - val_accuracy: 0.8134
Epoch 5/5
69/69 [==============================] - 190s 3s/step - loss: 0.3540 - accuracy: 0.8747 - val_loss: 0.5412 - val_accuracy: 0.8202
23/23 [==============================] - 51s 2s/step - loss: 0.5609 - accuracy: 0.8202
可见识别效率为0.82，也可以调参搭建更加复杂的模型。
