While this is not model builder, it should give you what you are looking for (python).
If you change the origin_x, origin_y, "MYFILE.csv", "layer_with_destinations","field_with_x_coordinate", and "field_with_y_coordinate" to the appropriate names this should give you a single table that has origin x, origin y, destination x and destination y fields which you can use for the input for "XY To Line"
import arcpy
import csv
cur = arcpy.UpdateCursor("layer_with_destinations")
destinations=[]
origin_x = #put your start point x coordinate here
origin_y = #put your start point y coordinate here
for i in cur:
x,y=i.getValue("field_with_x_coordinate"),i.getValue("field_with_y_coordinate")
destinations.append([x,y])
output_file = open("MYFILE.csv", "wb")#put your desired output csv file here including path
c = csv.writer(output_file)
c.writerow(["org_x", "org_y", "dest_x", "dest_y"])
for dest in destinations:
c.writerow([origin_x, origin_y, dest[0], dest[1]])
output_file.close()