hans

hans

HomeArchives

【Python】【Shell】【Caffe】训练集预处理 —— 数据增强 《很认真的讲讲Caffe》

#Python153
AI-generated summary
This code is for data augmentation in training sets using Python, Shell, and Caffe. It includes seven different methods for data augmentation. The code takes a directory name and a mode as input parameters and creates a new directory to save the augmented data. The first method is flipping the image, which can be randomly applied to the batch images. Other methods include channel shift, converting the image to grayscale, adjusting contrast, making the image lighter or darker, and adjusting saturation. The code uses the argparse module to parse the input parameters.

----------【2017.09.29】更新包含 7 种数据增强方法的代码 ----------------------------------------

#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Fri Sep 29 15:42:18 2017

@author: hans

http://blog.csdn.net/renhanchi
"""

import skimage
import skimage.io
import numpy as np
import matplotlib.pyplot as plt
import os
import argparse

import sys
reload(sys)
sys.setdefaultencoding('utf-8')

num = 0

def flip(image):
    return np.fliplr(image)

def channel_shift(x, limit=0.1, channel_axis=2):
    x = np.rollaxis(x, channel_axis, 0)
    min_x, max_x = np.min(x), np.max(x)
    channel_images = [np.clip(x_ch + np.random.uniform(-limit, limit), min_x, max_x) for x_ch in x]
    x = np.stack(channel_images, axis=0)
    x = np.rollaxis(x, 0, channel_axis + 1)
    return x

def gray(img):
    coef = np.array([[[0.114, 0.587, 0.299]]])
    gray = np.sum(img * coef, axis=2)
    img = np.dstack((gray, gray, gray))
    return img

def contrast(img, limit=0.3):
    alpha = 1.0 + np.random.uniform(-limit, limit)
    coef = np.array([[[0.114, 0.587, 0.299]]])
    gray = img * coef
    gray = (3.0 * (1.0 - alpha) / gray.size) * np.sum(gray)
    img = alpha * img + gray
    img = np.clip(img, 0., 1.)
    return img

def lighter(img):
    return skimage.exposure.adjust_gamma(img, 0.5)

def darker(img):
    return skimage.exposure.adjust_gamma(img, 2)

def saturation(img, limit=0.3):
    alpha = 1.0 + np.random.uniform(-limit, limit)
    coef = np.array([[[0.114, 0.587, 0.299]]])
    gray = img * coef
    gray = np.sum(gray, axis=2, keepdims=True)
    img = alpha * img + (1. - alpha) * gray
    img = np.clip(img, 0., 1.)
    return img

parser = argparse.ArgumentParser()

parser.add_argument(
        'n',
        type = str,
        help = """\
        directory name
        """
)

parser.add_argument(
        'm',
        type = str,
        default = 'flip',
        help = """\
        mode:
        flip(img),
        channel_shift(img, limit=0.1, channel_axis=2),
        gray(img),
        contrast(img, limit=0.3),
        lighter(img),
        darker(img),
        saturation(img, limit=0.3) 
        """
)

FLAGS = parser.parse_args()
mode = FLAGS.m
cla = FLAGS.n

dirpath = r'%s/' %cla
for dirname in os.listdir(dirpath):
    if os.path.isdir(r'%s%s' %(dirpath, dirname)): #判断是否是目录
        if not os.path.exists(r'%s_%s/%s/' %(cla, mode, dirname)): #判断镜像目录是否存在
            os.makedirs(r'%s_%s/%s/' %(cla, mode, dirname)) #不存在就新建目录
            for imagename in os.listdir(r'%s%s'%(dirpath, dirname)):
                num += 1
                print '%s saving %s_%s/%s/%s' %(num, cla, mode, dirname, imagename)
                image = os.path.join('%s%s/%s' % (dirpath, dirname, imagename))
                ori_Image = skimage.img_as_float(skimage.io.imread(image)).astype(np.float64)
                if mode == 'flip':
                    transform_image = flip(ori_Image)
                elif mode == 'channel_shift':
                    transform_image = channel_shift(ori_Image)
                elif mode == 'gray':
                    transform_image = gray(ori_Image)
                elif mode == 'contrast':
                    transform_image = contrast(ori_Image)
                elif mode == 'lighter':
                    transform_image = lighter(ori_Image)
                elif mode == 'darker':
                    transform_image = darker(ori_Image)
                elif mode == 'saturation':
                    transform_image = saturation(ori_Image)
                plt.imsave('%s_%s/%s/%s' %(cla, mode, dirname, imagename), transform_image, format='JPEG')

输入第一个参数是目录名，没有任何符号，就是目录名。第二个参数是数据增强模式名，没有任何其他内容，就是模式名.

有时候训练集太少，很快就过拟合.

增加训练集是必须的.

目录结构:

当前目录: flower, Mirror.py 等

flower/ : 各种类别花的目录

各种类别花的目录 / : 当前种类花的所有图片

执行后，在当前目录下创建新目录保存新数据

1. 镜像图片#

这个方法用或者不用看个人选择，因为在 caffe 数据层预处理方法中有一个 mirror 开关，这个开关是随机 mirror 当前 batch 图片。

layer {
  name: "data"
  type: "Data"
  top: "data"
  top: "label"
  include {
    phase: TRAIN
  }
  transform_param {
    crop_size: 227
    mean_file: ".binaryproto"
    mirror: true #这个就是随机对数据做镜像预处理的开关。
  }
  data_param {
    source: "train_lmdb_227"
    batch_size: 64
    backend: LMDB
  }

代码还是要放出来的。

#!/usr/bin/env python2
"""
Created on Fri Jul 21 11:08:23 2017

@author: hans
"""

import skimage
import skimage.io
import numpy as np
import matplotlib.pyplot as plt
import os

import sys
reload(sys)
sys.setdefaultencoding('utf-8') #ubuntu系统, windows下用gbk

cla = 'vegetable'

dirpath = r'%s/' %cla
for dirname in os.listdir(dirpath):
    if os.path.isdir(r'%s%s' %(dirpath, dirname)): #判断是否是目录
        if not os.path.exists(r'%s_mirror/%s/' %(cla, dirname)): #判断镜像目录是否存在
            os.makedirs(r'%s_mirror/%s/' %(cla, dirname)) #不存在就新建目录
            print "creat dir: %s" %dirname
            
        for imagename in os.listdir(r'%s%s'%(dirpath, dirname)):
            image = os.path.join('%s%s/%s' % (dirpath, dirname, imagename))
            oriMirror = skimage.img_as_float(skimage.io.imread(image)).astype(np.float64)
            imgMirror = np.fliplr(oriMirror)
            plt.imsave('%s_mirror/%s/%s' %(cla, dirname, imagename), imgMirror, format='JPEG')

2. 修改亮度#

#!/usr/bin/env python2
"""
Created on Fri Jul 21 11:08:23 2017

@author: hans
"""

import skimage
import skimage.io
import matplotlib.pyplot as plt
import numpy as np
import os

import sys
reload(sys)
sys.setdefaultencoding('utf-8')

cla = 'animal'
mode = 'lighter'

dirpath = r'%s/' %cla
for dirname in os.listdir(dirpath):
    if os.path.isdir(r'%s%s' %(dirpath, dirname)):
        if not os.path.exists(r'%s_%s/%s/' %(cla, mode, dirname)):
            os.makedirs(r'%s_%s/%s/' %(cla, mode, dirname))
            for imagename in os.listdir(r'%s%s'%(dirpath, dirname)):
                print 'saving %s_%s/%s/%s' %(cla, mode, dirname, imagename)
                image = os.path.join('%s%s/%s' % (dirpath, dirname, imagename))
                ori = skimage.img_as_float(skimage.io.imread(image)).astype(np.float32)
                img = skimage.exposure.adjust_gamma(ori, 0.5) # 小于1变亮，大于1变暗，跟上面mode匹配好
                plt.imsave('%s_%s/%s/%s' %(cla, mode, dirname, imagename), img, format='JPEG')

未完待续...

路径脚本#

将数据名，数据类型索引和从当前目录开始到该数据的路径保存到.txt , 并乱序.

#!/bin/sh

classes=(Anthurium asparagus_fern bamboo_palm Begonia cactus cape_jasmine Carnation Cherry_plum chrysanthemum)

for cla in flower flower_mirror
do
	num=0 #类别索引
	for class in ${classes[@]}
	do
		ls $cla/$class/* > $class.txt
		sed -i "s/$/ $num/g" $class.txt #末尾添加类别索引
		let num+=1
		cat $class.txt >> temp.txt
		rm $class.txt
	done
done
cat temp.txt | awk 'BEGIN{srand()}{print rand()"\t"$0}' | sort -k1,1 -n | cut -f2- > flower_train.txt #乱序
rm temp.txt
Loading...
Ownership of this post data is guaranteed by blockchain and smart contracts to the creator alone.