整流された線形単位を持つ1隠れ層ニューラルネットワーク

Question

私の目標は整流線形単位nn.relu()と1024個の隠れノードを持つ1隠れ層神経回路網を実装することです。

# These are all the modules we'll be using later. Make sure you can import them 
# before proceeding further. 
from __future__ import print_function 
import matplotlib.pyplot as plt 
import numpy as np 
import os 
import sys 
import tarfile 
from IPython.display import display, Image 
from scipy import ndimage 
from sklearn.linear_model import LogisticRegression 
from six.moves.urllib.request import urlretrieve 
from six.moves import cPickle as pickle 
from six.moves import range 
import tensorflow as tf 

url = 'https://commondatastorage.googleapis.com/books1000/' 
last_percent_reported = None 
data_root = '.' # Change me to store data elsewhere 

def download_progress_hook(count, blockSize, totalSize): 
    """A hook to report the progress of a download. This is mostly intended for users with 
    slow internet connections. Reports every 5% change in download progress. 
    """ 
    global last_percent_reported 
    percent = int(count * blockSize * 100/totalSize) 

    if last_percent_reported != percent: 
    if percent % 5 == 0: 
     sys.stdout.write("%s%%" % percent) 
     sys.stdout.flush() 
    else: 
     sys.stdout.write(".") 
     sys.stdout.flush() 

    last_percent_reported = percent 

def maybe_download(filename, expected_bytes, force=False): 
    """Download a file if not present, and make sure it's the right size.""" 
    dest_filename = os.path.join(data_root, filename) 
    if force or not os.path.exists(dest_filename): 
    print('Attempting to download:', filename) 
    filename, _ = urlretrieve(url + filename, dest_filename, reporthook=download_progress_hook) 
    print('\nDownload Complete!') 
    statinfo = os.stat(dest_filename) 
    if statinfo.st_size == expected_bytes: 
    print('Found and verified', dest_filename) 
    else: 
    raise Exception(
     'Failed to verify ' + dest_filename + '. Can you get to it with a browser?') 
    return dest_filename 

# If error in download get it here: http://yaroslavvb.com/upload/notMNIST/ 
train_filename = maybe_download('notMNIST_large.tar.gz', 247336696) 
test_filename = maybe_download('notMNIST_small.tar.gz', 8458043) 

num_classes = 10 
np.random.seed(133) 

def maybe_extract(filename, force=False): 
    root = os.path.splitext(os.path.splitext(filename)[0])[0] # remove .tar.gz 
    if os.path.isdir(root) and not force: 
    # You may override by setting force=True. 
    print('%s already present - Skipping extraction of %s.' % (root, filename)) 
    else: 
    print('Extracting data for %s. This may take a while. Please wait.' % root) 
    tar = tarfile.open(filename) 
    sys.stdout.flush() 
    tar.extractall(data_root) 
    tar.close() 
    data_folders = [ 
    os.path.join(root, d) for d in sorted(os.listdir(root)) 
    if os.path.isdir(os.path.join(root, d))] 
    if len(data_folders) != num_classes: 
    raise Exception(
     'Expected %d folders, one per class. Found %d instead.' % (
     num_classes, len(data_folders))) 
    print(data_folders) 
    return data_folders 

train_folders = maybe_extract(train_filename) 
test_folders = maybe_extract(test_filename) 

pickle_file = 'notMNIST.pickle' 

with open(pickle_file, 'rb') as f: 
    save = pickle.load(f,encoding='latin1') 
    train_dataset = save['train_dataset'] 
    train_labels = save['train_labels'] 
    valid_dataset = save['valid_dataset'] 
    valid_labels = save['valid_labels'] 
    test_dataset = save['test_dataset'] 
    test_labels = save['test_labels'] 
    del save # hint to help gc free up memory 
    print('Training set', train_dataset.shape, train_labels.shape) 
    print('Validation set', valid_dataset.shape, valid_labels.shape) 
    print('Test set', test_dataset.shape, test_labels.shape) 

image_size = 28 
num_labels = 10 

def reformat(dataset, labels): 
    dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32) 
    # Map 0 to [1.0, 0.0, 0.0 ...], 1 to [0.0, 1.0, 0.0 ...] 
    labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32) 
    return dataset, labels 
train_dataset, train_labels = reformat(train_dataset, train_labels) 
valid_dataset, valid_labels = reformat(valid_dataset, valid_labels) 
test_dataset, test_labels = reformat(test_dataset, test_labels) 
print('Training set', train_dataset.shape, train_labels.shape) 
print('Validation set', valid_dataset.shape, valid_labels.shape) 
print('Test set', test_dataset.shape, test_labels.shape) 

batch_size = 128 
hidden_nodes = 1024 

graph = tf.Graph() 
with graph.as_default(): 

    x_train = tf.placeholder(tf.float32, shape=(batch_size, image_size * image_size)) 
    y_ = tf.placeholder(tf.float32, shape=(batch_size, num_labels)) 
    x_valid = tf.constant(valid_dataset) 
    x_test = tf.constant(test_dataset) 

    hidden_layer = tf.contrib.layers.fully_connected(x_train,hidden_nodes) 

    logits = tf.contrib.layers.fully_connected(hidden_layer, num_labels, activation_fn=None) 
    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits,labels=y_)) 

    optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss) 

    train_prediction = tf.nn.softmax(logits) 
    valid_relu = tf.contrib.layers.fully_connected(x_valid,hidden_nodes) 
    valid_prediction = tf.nn.softmax(tf.contrib.layers.fully_connected(valid_relu,num_labels)) 

    test_relu = tf.contrib.layers.fully_connected(x_test,hidden_nodes, activation_fn=None) 
    test_prediction = tf.nn.softmax(tf.contrib.layers.fully_connected(test_relu,num_labels, activation_fn=None)) 

steps = 3001 

with tf.Session(graph=graph) as session: 
    tf.global_variables_initializer().run() 

    for step in range(steps): 
     # Selecting some portion within training data 
     # Note: Better to randomize dataset for Minibatch SGD 
     offset = (step * batch_size) % (train_labels.shape[0] - batch_size) 
     # Generate the Minibatch 
     batch_data = train_dataset[offset:(offset + batch_size), :] 
     batch_labels = train_labels[offset:(offset + batch_size), :] 
     # Feed the batch size to dict 
     feed_dict = {x_train: batch_data, y_:batch_labels} 
     _, l, prediction = session.run([optimizer, loss, train_prediction], feed_dict=feed_dict) 
     if(step % 500 == 0): 
      print("Minibatch Loss at step %d: %f"% (step, l)) 
      print("Minibatch accuracy: %.1f%%" % accuracy(prediction,batch_labels)) 
      print("Validation accuracy :%.1f%% "% accuracy(valid_prediction.eval(),valid_labels)) 



    print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels)) 

私は、次のいthis tutorial、それは私のコードよりも良い精度を得ました。

tf.contrib.layers.fully_connectedを隠しレイヤーとして使用しても同様の結果が得られたかったのですが、正しく行っていましたか？

EDIT：logits

でhidden_​​layerに

変更入力valid_relu、valid_prediction、test_relu、test_prediction

結果作り直し：

Minibatch Loss at step 0: 2.389448 
Minibatch accuracy: 5.5% 
Validation accuracy :8.2% 
Minibatch Loss at step 500: 0.342108 
Minibatch accuracy: 92.2% 
Validation accuracy :8.2% 
Minibatch Loss at step 1000: 0.543803 
Minibatch accuracy: 84.4% 
Validation accuracy :8.2% 
Minibatch Loss at step 1500: 0.299978 
Minibatch accuracy: 93.8% 
Validation accuracy :8.2% 
Minibatch Loss at step 2000: 0.294090 
Minibatch accuracy: 93.8% 
Validation accuracy :8.2% 
Minibatch Loss at step 2500: 0.333070 
Minibatch accuracy: 90.6% 
Validation accuracy :8.2% 
Minibatch Loss at step 3000: 0.365324 
Minibatch accuracy: 89.1% 
Validation accuracy :8.2% 
Test accuracy: 6.8% 

Answer 1

をあなたは右から始まりました。 Here'reいくつかの追加：

• あなたがtf.contrib.layers.fully_connectedのfaviourに手動FC層を退治しているので、同様wとbをドロップします。これは、これらの重みのために右の初期化を選ぶあなたの時間を節約します：

hidden_layer = tf.contrib.layers.fully_connected(x_train, hidden_nodes) 
logits = tf.contrib.layers.fully_connected(hidden_layer, num_labels, 
              activation_fn=None) 

• それもチュートリアルでは、右の定数としてグラフにあなたのデータセットを入れても、推論ノードを複製するために練習する悪いです。代わりに、valid_datasetとtest_datasetをfeed_dictと入力し、train_predictionと評価してください。

# BAD idea: this potentially large value is stored in the graph, can lead to OOM 
x_valid = tf.constant(valid_dataset) 
x_test = tf.constant(test_dataset) 
... 
# BAD idea: model duplication 
valid_relu = tf.contrib.layers.fully_connected(x_valid, hidden_nodes) 
valid_prediction = tf.nn.softmax(tf.matmul(valid_relu, w) + b) 
test_relu = tf.contrib.layers.fully_connected(x_test, hidden_nodes) 
test_prediction = tf.nn.softmax(tf.matmul(test_relu, w) + b) 

• またtensorflow.contribは実験的パッケージであることに注意してください。特に、fully_connected層はtf.layers.denseに「段階的に」なっています。同じ仕事をしていますが、そのAPIは安定していることが保証されていますが、fully_connectedは次期リリースで廃止される可能性があります。