function train_parallel(data, model, num_workers):
   shards = split_data(data, num_workers)

   for shard in shards (in parallel, one per GPU):
      local_gradients = compute_gradients(model, shard)
   combined_gradients = average(all local_gradients) # synchronize across work
 ers
   model.update(combined_gradients)
