i was writer one tensorflow yolo, But my loss has not been updated. Can someone check it for me?
my all code github link ：
my optimizer update code:
if ni <= nw:
xi = [0,nw]
# ？奇怪，为啥要获取这个？
# 这个东西大概意思是求nbs/tbs 的第ni个线性变化值，最小必须是1，然后会做取整操作
# 累计？累计了什么？
accumulate = max(1,np.interp(ni, xi, [1, nbs / total_batch_size]).round())
# 这是在干嘛？
# for j, x in enumerate(opt):
#通过_set_hyper调整lr和mom
# optimizer._set_hyper("learning_rate", np.interp(ni, xi, 0.0, hyp['lr0'] * lf(epoch)))
optimizer._set_hyper("momentum", np.interp(ni, xi, [hyp['warmup_momentum'], hyp['momentum']]))
# 是否要做缩放
if opt.multi_scale:
sz = random.randrange(imgsz * 0.5, imgsz * 1.5 * gs) // gs * gs
sf = sz / max(imgs.shape[2:]) # scale factor
if sf != 1:
ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:]]# new shape (stretched to gs-multiple)
imgs = tf.image.resize(imgs, ns , tf.image.ResizeMethod.BILINEAR,False)
#Forward
with tf.GradientTape() as gt:
pred = model(imgs)
loss, loss_items = compute_loss(pred,targets,model)
print(loss_items,end='')
print(loss)
if rank != -1 :
# gradient averaged between devices in DDP mode
# ddp模式下需要配置设备间的梯度平均值
loss *= opt.world_size
grads = gt.gradient(loss,model.trainable_variables)
optimizer.apply_gradients((grad, var) for (grad, var) in zip(grads,model.trainable_variables) if grad is not None) # 优化函数应用梯度进行优化
# optimizer.apply_gradients(zip(grads,model.trainable_variables)) # 不要使用trainable_variables**strong text**
my trian code
for epoch in range(start_epoch,epochs):
# 更新图片权重
if opt.image_weights:
# 生成索引
if rank in [-1, 0]:
cw = model.class_widths * (1 - maps) ** 2 /nc # 类权重(class weights)
iw = model.class_widths * (1 - maps) ** 2 /nc # 图片权重(image weights)
dataset.indices = random.choice(range(dataset.n),weights=id, k = dataset.n)
#如果是ddp模式(ddp大概是分布式训练？)，就需要配置广播
# if rank != -1 :
# indices = np.array[dataset.indices] if rank == 0 else np.zeros(dataset.n,dtype=np.int8)
mloss = np.zeros(5)
logger.info(('\n' + '%10s' * 9) % ('Epoch', 'gpu_mem', 'box', 'obj', 'cls', 'landmark', 'total', 'targets', 'img_size'))
# dataset len
pbar = range(nb)
if(rank in [-1,0]):
pbar = tqdm(pbar,total=nb)
for i in pbar:
# number integrated batches (since train start)
# 这个参大概意思是本次训练累计使用了多少图片了
ni = i + nb * epoch
# 通过批次数读取一次epochs所需要使用到的数据
(imgs, targets, paths)=[],[],[]
batch_index = 0
for imgi in range(i * batch_size , i * batch_size + batch_size):
# 下表从0开始，删一个
if(imgi < dl ):
img,target,path = dataset.__getitem__(imgi)
# 在getitem里改图片的shape有点困难，还是在外面改吧
if(opt.format=='NHWC'):
img = tf.transpose(img,perm=[1,2,0]).numpy()
imgs.append(img)
targets.append(target)
paths.append(path)
(imgs, targets, paths) = dataset.collate_fn(imgs,targets,paths)
imgs = np.array(imgs,dtype=np.float32) / 255.0 #将图片从uint8的0-255转化为float32的0到1
#预处理了图像一下
if ni <= nw:
xi = [0,nw]
# ？奇怪，为啥要获取这个？
# 这个东西大概意思是求nbs/tbs 的第ni个线性变化值，最小必须是1，然后会做取整操作
# 累计？累计了什么？
accumulate = max(1,np.interp(ni, xi, [1, nbs / total_batch_size]).round())
# 这是在干嘛？
# for j, x in enumerate(opt):
#通过_set_hyper调整lr和mom
# optimizer._set_hyper("learning_rate", np.interp(ni, xi, 0.0, hyp['lr0'] * lf(epoch)))
optimizer._set_hyper("momentum", np.interp(ni, xi, [hyp['warmup_momentum'], hyp['momentum']]))
# 是否要做缩放
if opt.multi_scale:
sz = random.randrange(imgsz * 0.5, imgsz * 1.5 * gs) // gs * gs
sf = sz / max(imgs.shape[2:]) # scale factor
if sf != 1:
ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:]]# new shape (stretched to gs-multiple)
imgs = tf.image.resize(imgs, ns , tf.image.ResizeMethod.BILINEAR,False)
#Forward
with tf.GradientTape() as gt:
pred = model(imgs)
loss, loss_items = compute_loss(pred,targets,model)
print(loss_items,end='')
print(loss)
if rank != -1 :
# gradient averaged between devices in DDP mode
# ddp模式下需要配置设备间的梯度平均值
loss *= opt.world_size
grads = gt.gradient(loss,model.trainable_variables)
optimizer.apply_gradients((grad, var) for (grad, var) in zip(grads,model.trainable_variables) if grad is not None) # 优化函数应用梯度进行优化
# optimizer.apply_gradients(zip(grads,model.trainable_variables)) # 不要使用trainable_variables
#?什么玩意
# tensorflow的混合精度学习。。后面再琢磨，现在不想
# scaler.scale(loss).backward()
# Optimize
# if ni % accumulate == 0 :
# scaler.step(optimizer)
# scaler.update()
# optimizer.zero_grad()
# if ema :
# ema.update(model)
# Print
if rank in [-1,0]:
mloss = (mloss * i + loss_items) / (i+1) # 更新当前平均loss
mem = '%.3G' % ( pynvml.nvmlDeviceGetMemoryInfo(handle).used / 1E9 if useGpu else 0)
s = ('%10s' * 2 + '%10.4g' * 7) % (
'%g/%g' % (epoch, epochs - 1), mem, *mloss, targets.shape[0], imgs.shape[-1] if opt.format == 'NCHW' else imgs.shape[1])
pbar.set_description(s)
# Plot
if plots and ni < 3:
f = save_dir / f'train_batch{ni}.jpg' # filename
f = ''
Thread(target=plot_images, args=(imgs, targets, paths, f, opt.format), daemon=True).start()
# if tb_writer:
# tb_writer.add_image(f, result, dataformats='HWC', global_step=epoch)
# tb_writer.add_graph(model, imgs) # add model to tensorboard
elif plots and ni == 3 and wandb:
wandb.log({"Mosaics": [wandb.Image(str(x), caption=x.name) for x in save_dir.glob('train*.jpg')]})
# 单次训练完成，清理掉img，paths和 targets
del imgs, targets, paths
# # 跑完一次epoch记得gc叫出来一下，做个深度清理
gc.collect()
model.save("mask_detector")