paulxiong / cervical Goto Github PK

$ cd yourdir/
$ git clone https://github.com/paulxiong/cervical
$ cd cervical
$ git checkout origin/add_final_medical_cervix -b add_final_medical_cervix
$ cd final_medical_cervix

$ rm -rf datasets/classify/*
$ git checkout ./datasets  
$ rm -rf datasets/origin_test/negative_test_images/

$ mkdir ./src/SEGMENT/kaggle-dsb2018/src/all_output
$ sudo cp  /opt/yunhai/final_medical_cervix/datasets/segment/stage1_train datasets/segment/stage1_train -a
$ sudo chmod 777 datasets/segment/stage1_train/ -R

$ sudo mkdir ./src/SEGMENT/kaggle-dsb2018/src/all_output/models_colour
$ sudo cp /opt/yunhai/final_medical_cervix/src/SEGMENT/kaggle-dsb2018/src/all_output/models_colour/single--2018-12-22-02-17-32.h5 ./src/SEGMENT/kaggle-dsb2018/src/all_output/models_colour/
$ sudo chmod a+rw ./src/SEGMENT/kaggle-dsb2018/src/all_output/models_colour -R

datasets/origin_test/negative_test_images/放入negitive图片, 然后执行下面脚本
$ ./run_slide_negitive_test.sh all

$ rm yourdir/yourdir/cervical/experiment/data/segmented/negative_test_npy/160/*
$ cp datasets/classify/npy/*  yourdir/cervical/experiment/data/segmented/negative_test_npy/160/

$ rm -rf datasets/classify/*

$ cp /opt/yunhai/final_medical_cervix/datasets/segment/test datasets/segment/ -a

datasets/origin_test/positive_test_images/放入positive图片, 然后执行下面脚本
$ ./run_slide_positive_test.sh all

$ rm -rf yourdir/cervical/experiment/data/segmented/positive_test_npy/160/*
$ cp datasets/classify/npy/* yourdir/cervical/experiment/data/segmented/positive_test_npy/160/ -a

链接：https://pan.baidu.com/s/1ktcTzUXE3NH9QYTL3rUHjw 
提取码：ad6s

阴性细胞团
├── glandular        3824
└── squamous-pm-two  4015

阳性细胞团
├── glandular-adcae  51
├── squamous-asc-h   3091
├── squamous-asc-us  2717

$ docker run -d --name='autokeras'  -v "$(pwd)":/app python:3.6.8-slim-jessie sleep 100h

$ docker exec -it -e LINES=$(tput lines) -e COLUMNS=$(tput cols) autokeras /bin/bash

docker里面执行
# apt-get update
# apt-get install vim git wget -y

# pip install autokeras==0.4.0  -i https://pypi.tuna.tsinghua.edu.cn/simple --no-cache-dir
# pip install keras==2.2.4 -i https://pypi.tuna.tsinghua.edu.cn/simple --no-cache-dir

docker里面执行
# curl https://raw.githubusercontent.com/keras-team/autokeras/master/examples/a_simple_example/mnist.py --output mnist.py

# python3.6 mnist.py
Using TensorFlow backend.
Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.
Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz
11493376/11490434 [==============================] - 118s 10us/step
Saving Directory: /tmp/autokeras_ZYZ97D
Preprocessing the images.
Preprocessing finished.

Initializing search.
Initialization finished.


+----------------------------------------------+
|               Training model 0               |
+----------------------------------------------+
Epoch-15, Current Metric - 0.978:  30%|██████▉                | 140/465 [00:27<01:08,  4.76 batch/s]

import PIL.Image as Image

file1 = 'my_data_batch'
def unpickle(file):
    import pickle
    with open(file, 'rb') as fo:
        dict = pickle.load(fo, encoding='iso-8859-1')
    return dict

dict_train_batch1 = unpickle(file1)
data_train_batch1 = dict_train_batch1.get('data')
labels = dict_train_batch1.get('labels')
img1 = dict_train_batch1['data'][10]
a = img1.reshape(3, 32, 32)

# 得到RGB通道
r = Image.fromarray(a[0]).convert('L')
g = Image.fromarray(a[1]).convert('L')
b = Image.fromarray(a[2]).convert('L')
image = Image.merge("RGB", (r, g, b))
# 显示图片
image.save("lambda.png", 'png')

input/
├── 1
│   ├── 0227_17P0603_1904202B_IMG027x003.JPG_1143_1929.png.png
│   ├── ...
│   └── 0464_17P0603_1904188B_IMG005x017.JPG_1167_177.png.png
└── 7
    ├── 0192_20190523_1812302_IMG004x019.JPG_1302_194.png.png
    ├── ...
    └── 0291_17P0603_1904852_IMG020x006.JPG_2096_704.png.png

from PIL import Image
import numpy as np
import os, pickle

def img21Dnumpy(imgpath):
    img = Image.open(imgpath,"r")
    w, h = img.size[0], img.size[1]
    if w != h:
        print("w!=h, skip !")
        return None
    img = img.resize((32, 32), Image.BILINEAR)
    r, g, b = img.split()
    r1, g1, b1 = np.asarray(r), np.asarray(g), np.asarray(b)
    r2, g2, b2 = r1.flatten(), g1.flatten(), b1.flatten()
    data = np.concatenate((r2, g2, b2))
    return data

def get_train_test(inputdir):
    x_train = None
    y_train = []
    imgpathlists = []
    labels = os.listdir(inputdir)
    for label in labels:
        path1 = os.path.join(inputdir, label)
        for imgname in os.listdir(path1):
            imgpath = os.path.join(path1, imgname)
            imgnpy = img21Dnumpy(imgpath)
            if imgnpy is None:
                continue
            y_train.append(imgname)
            imgpathlists.append(imgname)
            if x_train is None:
                x_train = np.array([img21Dnumpy(imgpath)])
            else:
                x_train = np.concatenate((x_train, [img21Dnumpy(imgpath)]))
    return x_train, y_train, imgpathlists

#生成data_batch文件
def make_data_batch(x, y, filenames):
    dic = {}
    dic['batch_label'] = "training batch 1 of 5"
    dic['labels'] = y
    dic['data'] = x
    dic['filenames'] = filenames
    #print(dic)
    with open('my_data_batch', 'wb') as f:
        pickle.dump(dic, f)

x, y, filenames = get_train_test("input")
make_data_batch(x, y, filenames)

http://2j592d3300.wicp.vip:8711/terminals/1

f774dcf5115d4004679015e9236344a4b430fe46da3ba449

# cd /notebooks/lambda/Faster-RCNN_TF/

python demo.py --model  model/VGGnet_fast_rcnn_iter_70000.ckpt

DEV=GPU
DEV_ID=0
TRAIN_IMDB="voc_2007_trainval"
TEST_IMDB="voc_2007_test"
PT_DIR="pascal_voc"
ITERS=70000
EXTRA_ARGS=""


python ./tools/train_net.py --device ${DEV} --device_id ${DEV_ID} \
  --weights data/pretrain_model/VGG_imagenet.npy \
  --imdb ${TRAIN_IMDB} \
  --iters ${ITERS} \
  --cfg experiments/cfgs/faster_rcnn_end2end.yml \
  --network VGGnet_train \
  ${EXTRA_ARGS}
root@a5a5ba0db669:

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Oct 28 16:34:02 2019

@author: yfq
"""

from keras import Sequential, initializers, optimizers
from keras.layers import Activation, Dense
import numpy as np
import pylab as pl
from IPython import display
from keras.callbacks import Callback
from keras.datasets import mnist
import keras
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Dense, Dropout, Flatten

#定义回调函数
class DrawCallback(Callback):
    #初始化
    def __init__(self, runtime_plot=True):

        self.init_loss = None
        self.init_val_loss = None
        self.init_acc = None
        self.init_val_acc = None
        self.runtime_plot = runtime_plot
        
        self.xdata = []
        self.ydata = []
        self.ydata2 = []
        self.ydata3 = []
        self.ydata4 = []
    #画图
    def _plot(self, epoch=None):
        epochs = self.params.get("epochs")
        pl.subplot(121)
        pl.ylim(0, int(self.init_loss*1.2))
        pl.xlim(0, epochs)
        pl.plot(self.xdata, self.ydata,'r', label='loss')
        pl.plot(self.xdata, self.ydata2, 'b--', label='val_loss')
        pl.xlabel('Epoch {}/{}'.format(epoch or epochs, epochs))
        pl.ylabel('Loss {:.4f}'.format(self.ydata[-1]))
        pl.legend()
        pl.title('loss')
        
        pl.subplot(122)
        pl.ylim(0, 1.2)
        pl.xlim(0, epochs)
        pl.plot(self.xdata, self.ydata3,'r', label='acc')
        pl.plot(self.xdata, self.ydata4, 'b--', label='val_acc')
        pl.xlabel('Epoch {}/{}'.format(epoch or epochs, epochs))
        pl.ylabel('Loss {:.4f}'.format(self.ydata[-1]))
        pl.legend()
        pl.title('acc')
    #清屏
    def _runtime_plot(self, epoch):
        self._plot(epoch)
        
        display.clear_output(wait=True)
        display.display(pl.gcf())
        pl.gcf().clear()
    #显示窗口
    def plot(self):
        self._plot()
        pl.show()
    #获取keras训练参数
    def on_epoch_end(self, epoch, logs = None):
        logs = logs or {}
        epochs = self.params.get("epochs")
        loss = logs.get("loss")
        val_loss = logs.get("val_loss")
        acc = logs.get("acc")
        val_acc = logs.get("val_acc")
        #转为字符型，限制数据有效长度
        epochs_str = str(epochs)[0:6]
        loss_str = str(loss)[0:6]
        val_loss_str = str(val_loss)[0:6]
        acc_str = str(acc)[0:6]
        val_acc_str = str(val_acc)[0:6]
        #数据写入txt
        f = open('logs_r/record.txt','a')
        f.write('epochs:{}-loss:{}-val_loss:{}-acc:{}-val_acc{}'.format(epochs_str,loss_str,val_loss_str,acc_str,val_acc_str))
        f.write('\n')
        f.close()  
        #更新xdata/ydata
        if self.init_loss is None:
            self.init_loss = loss
            self.init_val_loss = val_loss
        self.xdata.append(epoch)
        self.ydata.append(loss)
        self.ydata2.append(val_loss)
        self.ydata3.append(acc)
        self.ydata4.append(val_acc)
        if self.runtime_plot:
            self._runtime_plot(epoch)
            
#搭建keras需要的东西
def viz_keras_fit(runtime_plot=False):
    d = DrawCallback(runtime_plot = runtime_plot)#实例化回调函数
    #获取训练/测试数据
    (x_train, y_train), (x_test, y_test) = mnist.load_data()
    x_train = x_train.reshape(-1,28,28,1)
    x_test = x_test.reshape(-1,28,28,1)
    input_shape = (28,28,1)
    x_train = x_train/255
    x_test = x_test/255
    y_train = keras.utils.to_categorical(y_train,10)
    y_test = keras.utils.to_categorical(y_test,10)
    #较少计算量
    x_train = x_train[0:600,:,:,:]
    x_test = x_test[0:100,:,:,:]
    y_train = y_train[0:600,:]
    y_test = y_test[0:100,:]
    #定义网络
    model = Sequential()
    model.add(Conv2D(filters=32, kernel_size=(3,3),
                activation='relu', input_shape=input_shape,
                name='conv1'))
    model.add(Conv2D(64,(3,3),activation='relu',name='conv2'))
    model.add(MaxPooling2D(pool_size=(2,2),name='pool2'))
    model.add(Dropout(0.25,name='dropout1'))
    model.add(Flatten(name='flat1'))
    model.add(Dense(128,activation='relu'))
    model.add(Dropout(0.5,name='dropout2'))
    model.add(Dense(10,activation='softmax',name='output'))
    #编译网络
    model.compile(loss=keras.losses.categorical_crossentropy,
             optimizer=keras.optimizers.Adadelta(),
             metrics=['accuracy'])
    #训练模型
    model.fit(x = x_train,
             y = y_train,
             epochs=100,
             verbose=0,
              validation_data=(x_test, y_test),
             callbacks=[d]) # verbose=1可显示训练过程，validation_data加入测试数据，否则有些参数不显示，callbacks指定回调函数
    return d

viz_keras_fit(runtime_plot=True)

$ ls /opt/lambda_workspace/datasets/ -l
total 4
drwxrwxr-x 4 atec atec  24 Jun  6 15:30 0606
-rw-rw-r-- 1 atec atec 982 Jun 11 15:35 README.md

$ ls /opt/zhuoyao_workspace/medical_ai/datasets/origin_train/                                                                                                                                                
$ ls /opt/zhuoyao_workspace/medical_ai/datasets/origin_annotation/       

创建如下目录结构的目录，把对应的图片拷贝到名字对应的目录，执行generate_npy.py就生成了npy
train_datasets/
├── HPV
├── HSIL
├── LSIL
├── Norm
└── SCC