tensorflow 合并模型

在这里存个备份，还有些问题没有解决。

raise ValueError("GraphDef cannot be larger than 2GB.")

记录一些思路好了。现在是没有生成.meta文件，爆掉应该是因为所有的变量都加载到了默认图里。

也就是说我处理完checkpoint 0 之后开始处理checkpoint1,但是checkpoint0的那些变量还是存在的...所以越来越多？

目前有两个想法，第一个想法是是受TensorFlow极简教程：创建、保存和恢复机器学习模型  中启发，用多个saver，每个saver指定要搞的图（但是这样好像要每个checkpoint都是不同的saver才有意义？）

第二个想法是，每次save完变量之后，将图恢复成默认状态（可以把图中所有变量清空。。

想法二大失败：

会遇到if self.stack[-1] is not default: │ IndexError: list index out of range   的问题。。

根据 reset_default_graph awkwardly breaks graph nesting        

中提到了。。。reset_default_graph本身就不舍被设计成放在graph中清空变量用的。。。然后tf的代码也写得很不友好。。。没有 指明这个错误的原因。。。

For historical context, `tf.reset_default_graph()` was never designed to be used with `with g.as_default():` context managers. I think the proper fix here is to make `tf.reset_default_graph()` fail with an informative error message when used inside a `with g.as_default():` context. I think this could be done by checking that `ops._default_graph_stack` is empty before resetting.

import sys, getopt
import argparse
import tensorflow as tf
import os
import shutil
import numpy as np

# fix out ,not log_
def fix_var_name(id,var_name): 
    prefix = var_name[0:3]
    if id<10:
      suffix = var_name[4:]
    if id>=10 and id<100:
      suffix = var_name[5:]
    if id>=100 and id<1000:
      suffix = var_name[6:]
    if id>=1000 and id<10000:
      suffix = var_name[7:]
    ret = prefix + str(id+1) + suffix
    print('id=%d var_name=%s prefix=%s suffix=%s ret=%s' %(id,var_name,prefix,suffix,ret))
    return ret
# only concat full_link_layer 
def merge_full_link_layer(checkpoint_list,dry_run=False):
    with tf.Session() as sess:
      log_num = len(checkpoint_list) # a int range [0,1000)
      print("log_num:%d"%log_num)
      for var_name,_ in tf.contrib.framework.list_variables('log_0'):
        if not var_name.startswith('out'):
          var_tmp = tf.contrib.framework.load_variable('log_0',var_name)
          var = tf.Variable(var_tmp,name=var_name)
          continue
        print("var_name:%s"%var_name)
        for id in range(0,log_num): 
          # need to change the string  out0->out1,out2,out3 ... out15
          if id!=0:
            var_name = fix_var_name(id-1,var_name)
          checkpoint_dir = 'log_'+str(id)
          print('checkpoint_dir:%s'%checkpoint_dir)

       # for id,checkpoint_dir in enumerate(checkpoint_list):
       #  var_name = fix_var_name(id+1,var_name)
          var_tmp = tf.contrib.framework.load_variable(checkpoint_dir,var_name)
          #print("type(var_tmp):%s"%type(var_tmp))
       #   print(var_tmp)
          if 'weights' in var_name:
            if 'Momentum' in var_name:
              if id == 0:
                mom_weights = var_tmp
                #print("mom_weights:%s"%type(mom_weights))
              else:
                mom_weights = np.concatenate((mom_weights,var_tmp),axis=1)
            else:
              if id == 0:
                weights = var_tmp
              else:
                weights = np.concatenate((weights,var_tmp),axis=1)
          else:
            if 'Momentum' in var_name:
              if id == 0:
                mom_biases = var_tmp
              else:
                mom_biases = np.concatenate((mom_biases,var_tmp),axis=0)

            else:
              if id == 0:
                biases = var_tmp
              else: 
                biases = np.concatenate((biases,var_tmp),axis=0)
        if not dry_run:
            flag1 = 'weights' in var_name
            flag2 = 'Momentum' in var_name
            if flag1 and flag2:
              mom_weights = tf.Variable(mom_weights, name='out/weights/Momentum' )
            if flag1 and not flag2:
              weights = tf.Variable(weights,name='out/weights')
            if not flag1 and flag2:
              mom_biases = tf.Variable(mom_biases,name='out/biases/Momentum')
            if not flag1 and not flag2:
              biases = tf.Variable(biases,name='out/biases')
      if not dry_run:
        print("writer running")
        #writer = tf.summary.FileWriter('./graphs', sess.graph)
        saver = tf.train.Saver()
        #sess.run(tf.global_variables_initializer())
        saver.save(sess,'./final_16_out',write_meta_graph=False)
    #writer.close()
def merge_ckpt(checkpoint_dir,  dry_run=False):
  merge_full_link_layer(checkpoint_dir,False)
def get_dir():
  checkpoint_list=[]
  dir_list = os.listdir('./')
  for line in dir_list:
    if line.startswith('log') and os.path.isdir(line):
      checkpoint_list.append(line)
  return checkpoint_list 
def main():
  os.environ['CUDA_VISIBLE_DEVICES']="" 
  checkpoint_dir = get_dir()
  #checkpoint_dir = ['log_0','log_1','log_2']
  print (checkpoint_dir)
  merge_ckpt(checkpoint_dir, dry_run=False)
if __name__ == '__main__':
  main()

嘛。。先不管了。。。据数据那边说已经够用了。下面是最终版本,没有合并动量，因为对验证没有作用。

import sys, getopt
import argparse
import tensorflow as tf
import os
import shutil
import numpy as np

# fix out ,not log_
def fix_var_name(id,var_name): 
    prefix = var_name[0:3]
    if id<10:
      suffix = var_name[4:]
    if id>=10 and id<100:
      suffix = var_name[5:]
    if id>=100 and id<1000:
      suffix = var_name[6:]
    if id>=1000 and id<10000:
      suffix = var_name[7:]
    ret = prefix + str(id+1) + suffix
    print('id=%d var_name=%s prefix=%s suffix=%s ret=%s' %(id,var_name,prefix,suffix,ret))
    return ret
# only concat full_link_layer
def merge_full_link_layer(checkpoint_list):
    with tf.Session() as sess:
      log_num = len(checkpoint_list) # a int range [0,1000)
      print("log_num:%d"%log_num)
      for var_name,_ in tf.contrib.framework.list_variables('log_0'):
        if not var_name.startswith('out'):
          var_tmp = tf.contrib.framework.load_variable('log_0',var_name)
          var = tf.Variable(var_tmp,name=var_name)
          continue
        if 'Momentum' in var_name:
          continue
        print("var_name:%s"%var_name)
        for id in range(0,log_num): 
          # need to change the string  out0->out1,out2,out3 ... out15
          if id!=0:
            var_name = fix_var_name(id-1,var_name)
          checkpoint_dir = 'log_'+str(id)
          print('checkpoint_dir:%s'%checkpoint_dir)
          var_tmp = tf.contrib.framework.load_variable(checkpoint_dir,var_name)
          if 'weights' in var_name:
            if 'Momentum' in var_name:
              if id == 0:
                mom_weights = var_tmp
              else:
                mom_weights = np.concatenate((mom_weights,var_tmp),axis=1)
            else:
              if id == 0:
                weights = var_tmp
              else:
                weights = np.concatenate((weights,var_tmp),axis=1)
          else:
            if 'Momentum' in var_name:
              if id == 0:
                mom_biases = var_tmp
              else:
                mom_biases = np.concatenate((mom_biases,var_tmp),axis=0)
            else:
              if id == 0:
                biases = var_tmp
              else: 
                biases = np.concatenate((biases,var_tmp),axis=0)
        flag1 = 'weights' in var_name
        flag2 = 'Momentum' in var_name
        if flag1 and not flag2:
          weights = tf.Variable(weights,name='out/weights')
        if not flag1 and not flag2:
          biases = tf.Variable(biases,name='out/biases')
      print("writer running")
        #writer = tf.summary.FileWriter('./graphs', sess.graph)
      saver = tf.train.Saver()
      sess.run(tf.global_variables_initializer())
      saver.save(sess,'./final_result',write_meta_graph=False)
    #writer.close()
def get_dir():
  checkpoint_list=[]
  dir_list = os.listdir('./')
  for line in dir_list:
    if line.startswith('log') and os.path.isdir(line):
      checkpoint_list.append(line)
  return checkpoint_list 
def main():
  os.environ['CUDA_VISIBLE_DEVICES']="" 
  checkpoint_dir = get_dir()
  # get_dir return the all the log_dir in './'  the log_dir format is 'log_%d',such as log_0,log_1


  #checkpoint_dir=['log_0','log_1','log_2']
  #checkpoint_dir = ['log_0','log_1','log_2','log_3','log_4','log_5','log_6','log_7','log_8','log_9','log_10','log_11']
  print (checkpoint_dir)
  merge_full_link_layer(checkpoint_dir)
if __name__ == '__main__':
  main()

20170822update:去掉了卷基层的动量，添加了一些超参

import sys, getopt
import argparse
import tensorflow as tf
import os
import shutil
import numpy as np




tf.flags.DEFINE_string('fc_prefix', 'out',
                               """the prefix of full_link_layer output name """)




FLAGS = tf.flags.FLAGS
# fix out ,not log_
def fix_var_name(id,var_name):
    len = len(FLAGS.fc_prefix)
    prefix = var_name[0:len]
    if id<10:
      suffix = var_name[len+1:]
    if id>=10 and id<100:
      suffix = var_name[len+2:]
    if id>=100 and id<1000:
      suffix = var_name[len+3:]
    if id>=1000 and id<10000:
      suffix = var_name[len+4:]
    ret = prefix + str(id+1) + suffix
    print('id=%d var_name=%s prefix=%s suffix=%s ret=%s' %(id,var_name,prefix,suffix,ret))
    return ret
# only concat full_link_layer
def merge_full_link_layer(checkpoint_list):
    with tf.Session() as sess:
      log_num = len(checkpoint_list) # a int range [0,1000)
      print("log_num:%d"%log_num)
      for var_name,_ in tf.contrib.framework.list_variables('log_0'):
        if 'Momentum' in var_name:
          continue
        if not var_name.startswith(FLAGS.fc_prefix):
          var_tmp = tf.contrib.framework.load_variable('log_0',var_name)
          var = tf.Variable(var_tmp,name=var_name)
          continue
        print("var_name:%s"%var_name)
        for id in range(0,log_num): 
          # need to change the string  out0->out1,out2,out3 ... out15
          if id!=0:
            var_name = fix_var_name(id-1,var_name)
          checkpoint_dir = 'log_'+str(id)
          print('checkpoint_dir:%s'%checkpoint_dir)
          var_tmp = tf.contrib.framework.load_variable(checkpoint_dir,var_name)
          if 'weights' in var_name:
            if 'Momentum' in var_name:
              if id == 0:
                mom_weights = var_tmp
              else:
                mom_weights = np.concatenate((mom_weights,var_tmp),axis=1)
            else:
              if id == 0:
                weights = var_tmp
              else:
                weights = np.concatenate((weights,var_tmp),axis=1)
          else:
            if 'Momentum' in var_name:
              if id == 0:
                mom_biases = var_tmp
              else:
                mom_biases = np.concatenate((mom_biases,var_tmp),axis=0)
            else:
              if id == 0:
                biases = var_tmp
              else: 
                biases = np.concatenate((biases,var_tmp),axis=0)
        flag1 = 'weights' in var_name
        flag2 = 'Momentum' in var_name
        if flag1 and not flag2:
          weights = tf.Variable(weights,name='%s/weights'%FLAGS.fc_prefix)
        if not flag1 and not flag2:
          biases = tf.Variable(biases,name='%s/biases'%FLAGS.fc_prefix)
      print("writer running")
        #writer = tf.summary.FileWriter('./graphs', sess.graph)
      saver = tf.train.Saver()
      sess.run(tf.global_variables_initializer())
      saver.save(sess,'./final_result',write_meta_graph=False)
    #writer.close()
def get_dir():
  checkpoint_list=[]
  dir_list = os.listdir('./')
  for line in dir_list:
    if line.startswith('log') and os.path.isdir(line):
      checkpoint_list.append(line)
  return checkpoint_list 
def main():
  os.environ['CUDA_VISIBLE_DEVICES']="" 
  checkpoint_dir = get_dir()
  # get_dir return the all the log_dir in './'  the log_dir format is 'log_%d',such as log_0,log_1


  #checkpoint_dir=['log_0','log_1','log_2']
  #checkpoint_dir = ['log_0','log_1','log_2','log_3','log_4','log_5','log_6','log_7','log_8','log_9','log_10','log_11']
  print (checkpoint_dir)
  merge_full_link_layer(checkpoint_dir)
if __name__ == '__main__':
  main()