{"id":170,"date":"2014-07-09T20:53:57","date_gmt":"2014-07-09T11:53:57","guid":{"rendered":"https:\/\/www.dogrow.net\/nnet\/?p=170"},"modified":"2025-06-11T22:50:29","modified_gmt":"2025-06-11T13:50:29","slug":"blog13","status":"publish","type":"post","link":"https:\/\/www.dogrow.net\/nnet\/blog13\/","title":{"rendered":"(13) cuda-convnet\u7528MNIST\u30c7\u30fc\u30bf\u3092\u4f5c\u308b(\u305d\u306e3)"},"content":{"rendered":"

(12) cuda-convnet\u7528MNIST\u30c7\u30fc\u30bf\u3092\u4f5c\u308b(\u305d\u306e2)<\/a> \u3067\u306fMNIST\u5b66\u7fd2\u30c7\u30fc\u30bf\u30d5\u30a1\u30a4\u30eb\u304b\u3089 cuda-convnet\u5165\u529b\u7528\u306e\u5b66\u7fd2\u30c7\u30fc\u30bf\u60c5\u5831\u30d5\u30a1\u30a4\u30eb batches.meta<\/span> \u3092\u4f5c\u6210\u3057\u3066\u307f\u305f\u3002
\n\u4eca\u56de\u306f\u5b66\u7fd2\u30c7\u30fc\u30bf\u3001\u30c6\u30b9\u30c8\u30c7\u30fc\u30bf\u305d\u306e\u3082\u306e\u3092\u683c\u7d0d\u3059\u308b data_batch_n<\/span> \u3092\u4f5c\u6210\u3057\u3066\u307f\u308b\u3002<\/p>\n

cifar10\u306e data_batche_1 \u306e\u4e2d\u8eab\u306f\u3053\u3093\u306a\u611f\u3058\u3002<\/p>\n

>>> import cPickle as cp\r\n>>> d = cp.load(open('data_batch_1'))\r\n>>> d.keys()\r\n['batch_label', 'labels', 'data', 'filenames']\r\n>>> d['batch_label']\r\n'training batch 1 of 5'\r\n>>> d['labels']\r\n[6, 9, 9, 4, 1, 1, 2, 7, 8, 3, 4, \r\n ...,\r\n 9, 2, 2, 1, 6, 3, 9, 1, 1, 5]\r\n>>> d['data']\r\narray([[ 59, 154, 255, ...,  71, 250,  62],\r\n       [ 43, 126, 253, ...,  60, 254,  61],\r\n       ...,\r\n       [ 84, 142,  83, ...,  69, 255, 130],\r\n       [ 72, 144,  84, ...,  68, 254, 131]], dtype=uint8)\r\n>>> d['filenames']\r\n['leptodactylus_pentadactylus_s_000004.png', 'camion_s_000148.png', 'tipper_truck_s_001250.png', 'american_elk_s_001521.png',\r\n ...,\r\n , 'car_s_002296.png', 'estate_car_s_001433.png', 'cur_s_000170.png']\r\n<\/pre>\n
data_batche_n \u306e\u4e2d\u306b\u683c\u7d0d\u3055\u308c\u3066\u3044\u308b\u30c7\u30fc\u30bf\u306f4\u7a2e\u985e\u3042\u308b\u3002
\n1) batch_label<\/span> : \u30c7\u30fc\u30bf\u30d5\u30a1\u30a4\u30eb\u306e\u8aac\u660e
\n2) labels<\/span> : \u6b63\u89e3\u30e9\u30d9\u30eb
\n3) data<\/span> : \u753b\u50cf\u30c7\u30fc\u30bf
\n4) filenames<\/span> : \u5143\u306e\u753b\u50cf\u30d5\u30a1\u30a4\u30eb\u540d<\/p>\n
\u4f5c\u6210\u3057\u305f data_batche_n\u4f5c\u6210\u95a2\u6570\u306f\u3053\u3061\u3089\u3002
\n \u62d9\u3044Python\u30b9\u30ad\u30eb\u3067\u8a66\u884c\u932f\u8aa4\u3057\u306a\u304c\u3089\u66f8\u3044\u305f\u306e\u3067\u7121\u99c4\u304c\u3044\u3063\u3071\u3044\u3042\u308b\u304b\u3082\u2026<\/span><\/p>\n
import cPickle as cp\r\nimport numpy as np\r\nimport Image\r\nimport struct\r\n\r\ndef make_data_batch_all( train_img, train_lbl, test_img, test_lbl, nDataPerBatch ):\r\n    iBatch = 1\r\n    iBatch = make_data_batch( train_img, train_lbl, nDataPerBatch, iBatch, 'training' )\r\n    iBatch = make_data_batch( test_img,  test_lbl,  nDataPerBatch, iBatch, 'testing' )\r\n\r\ndef make_data_batch( img, lbl, nDataPerBatch, iBatch, batchLabel ):\r\n    fpImg = open( img, 'rb' )\r\n    fpLbl = open( lbl, 'rb' )\r\n    headerImg = fpImg.read( 4 * 4 )\r\n    headerLbl = fpLbl.read( 4 * 2 )\r\n\r\n    headerImg_up = struct.unpack('>4i', headerImg)\r\n    headerLbl_up = struct.unpack('>2i', headerLbl)\r\n\r\n    nImg  = headerImg_up[1]\r\n    img_w = headerImg_up[2]\r\n    img_h = headerImg_up[3]\r\n    nPixelsOf1img = img_w * img_h\r\n\r\n    print 'img : # of image             : %d' % nImg\r\n    print 'img : image width            : %d' % img_w\r\n    print 'img : image height           : %d' % img_h\r\n    print 'img : # of pixels in a image : %d' % nPixelsOf1img\r\n\r\n    nBatchMax = np.ceil(np.float32(nImg) \/ np.float32(nDataPerBatch))\r\n    fmtImg  = '%dB' % nPixelsOf1img\r\n    save_data = []\r\n    save_filenames = []\r\n    save_labels = []\r\n    nOutData = 0\r\n\r\n    for iImg in range(0,nImg):\r\n        if (iImg % 1000 == 0) or (iImg == nImg-1):\r\n            print 'now processing image #%d' % iImg\r\n        # append data<\/span>\r\n        data = fpImg.read( nPixelsOf1img )\r\n        data = struct.unpack(fmtImg, data)\r\n        data = np.asarray( data ).astype('uint8')\r\n        save_data.append(data.T.flatten('C'))\r\n        nOutData = nOutData + 1\r\n        # append filenames<\/span>\r\n        save_filenames.append('')\r\n        # append labels<\/span>\r\n        data = fpLbl.read(1)\r\n        data = struct.unpack('B', data)\r\n        save_labels.append(data[0])\r\n        # save data_batch_n file<\/span>\r\n        if ((iImg > 0) and (iImg % nDataPerBatch == nDataPerBatch-1)) or (iImg == nImg-1):\r\n           fname = 'data_batch_%d' % iBatch\r\n           fpOut = open( fname, 'w+' )\r\n           label = '%s batch %d of %d' % (batchLabel, iBatch, nBatchMax)\r\n           save_data = np.reshape(save_data, (nOutData, img_w, img_h))\r\n           save_data = np.uint8(save_data)\r\n           save_data = save_data.swapaxes(0,2)\r\n           save_data = np.reshape(save_data, (img_w * img_h, nOutData))\r\n           dic = {'batch_label':label, 'data':save_data, 'labels':save_labels, 'filenames':save_filenames}\r\n           cp.dump( dic, fpOut )\r\n           fpOut.close()\r\n           iBatch = iBatch + 1\r\n           # clear data<\/span>\r\n           save_data = []\r\n           save_filenames = []\r\n           save_labels = []\r\n           nOutData = 0\r\n    fpImg.close()\r\n    fpLbl.close()\r\n    return iBatch\r\n<\/pre>\n
\u4f5c\u6210\u3057\u305f data_batch_n \u3092\u78ba\u8a8d\u3057\u3066\u307f\u308b\u3002
\n\u5143\u753b\u50cf\u30d5\u30a1\u30a4\u30eb\u540d\u306f\u5b58\u5728\u3057\u306a\u3044\u306e\u3067\u7a7a\u6587\u5b57\u3068\u3057\u3066\u3044\u308b\u3002<\/p>\n
>>> import cPickle as cp\r\n>>> d = cp.load(open('data_batch_1'))\r\n>>> d.keys()\r\n['batch_label', 'labels', 'data', 'filenames']\r\n>>> d['batch_label']\r\n'training batch 1 of 6'\r\n>>> d['labels']\r\n[5, 0, 4, 1, 9, 2, 1, 3, 1, 4, 3, 5,\r\n ...,\r\n 7, 2, 7, 3, 7, 4, 0, 5, 8, 6, 9, 7]\r\n>>> d['data']\r\narray([[0, 0, 0, ..., 0, 0, 0],\r\n       [0, 0, 0, ..., 0, 0, 0],\r\n       ...,\r\n       [0, 0, 0, ..., 0, 0, 0],\r\n       [0, 0, 0, ..., 0, 0, 0]], dtype=uint8)\r\n>>> d['filenames']\r\n['', '', '', '', '', '', '', '', '',\r\n '', '', '', '', '', '', '', '', '']\r\n<\/pre>\n
OK\u305d\u3046\u3060\uff01<\/span><\/p>\n
\u6b21\u56de\u300c(14) cuda-convnet\u3067MNIST\u81ea\u52d5\u8a8d\u8b58(\u305d\u306e1)\u300d\u3067\u306f\u3001\u4f5c\u6210\u6e08\u307f\u306e batches.meta <\/span>\u3068 data_batch_n <\/span>\u3092\u4f7f\u7528\u3057\u3066 cuda-convnet<\/span> \u3092\u52d5\u304b\u3057\u3066\u307f\u307e\u3059\u3002<\/a><\/p>\n
\n","protected":false},"excerpt":{"rendered":"
(12) cuda-convnet\u7528MNIST\u30c7\u30fc\u30bf\u3092\u4f5c\u308b(\u305d\u306e2) \u3067\u306fMNIST\u5b66\u7fd2\u30c7\u30fc\u30bf\u30d5\u30a1\u30a4\u30eb\u304b\u3089 cuda-convnet\u5165\u529b\u7528\u306e\u5b66\u7fd2\u30c7\u30fc\u30bf\u60c5\u5831\u30d5\u30a1\u30a4\u30eb batches.meta \u3092\u4f5c\u6210\u3057\u3066\u307f\u305f\u3002 \u4eca\u56de\u306f\u5b66\u7fd2\u30c7\u2026 \u7d9a\u304d\u3092\u8aad\u3080 »<\/a><\/span><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[8,10,18,2],"tags":[],"class_list":["post-170","post","type-post","status-publish","format-standard","hentry","category-cuda","category-cuda-convnet","category-mnist","category-2"],"views":2962,"amp_enabled":true,"_links":{"self":[{"href":"https:\/\/www.dogrow.net\/nnet\/wp-json\/wp\/v2\/posts\/170","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.dogrow.net\/nnet\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.dogrow.net\/nnet\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.dogrow.net\/nnet\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.dogrow.net\/nnet\/wp-json\/wp\/v2\/comments?post=170"}],"version-history":[{"count":19,"href":"https:\/\/www.dogrow.net\/nnet\/wp-json\/wp\/v2\/posts\/170\/revisions"}],"predecessor-version":[{"id":2545,"href":"https:\/\/www.dogrow.net\/nnet\/wp-json\/wp\/v2\/posts\/170\/revisions\/2545"}],"wp:attachment":[{"href":"https:\/\/www.dogrow.net\/nnet\/wp-json\/wp\/v2\/media?parent=170"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.dogrow.net\/nnet\/wp-json\/wp\/v2\/categories?post=170"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.dogrow.net\/nnet\/wp-json\/wp\/v2\/tags?post=170"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}