Reading through the code of QGIS is your friend. I had this same task, found your question, and was also stumped. Then I looked at line 469 of
https://github.com/qgis/Quantum-GIS/blob/master/src/app/qgsdiagramproperties.cpp
Which has a complete implementation driven by the interface in QGIS.
Here is my python test implementation. It obviously works on my data as a test case.
#Load the diagram automatically
l = balayer
#print l.diagramRenderer().rendererName()
#line numbers in comments refer to
# https://github.com/qgis/Quantum-GIS/blob/master/src/app/qgsdiagramproperties.cpp
diagram = QgsPieDiagram() #500
ds = QgsDiagramSettings() #507
ds.font = QFont( "Helvetica", 12 ) #508
ds.transparency = 0 #509 - not transparent...
#dColors = dict() #511 - keys are the field numbers, values are the QColors
#categoryAttributes = dict() #512 - field numbers of fields
dColors = {4:QColor("cyan"), 5:QColor("darkCyan"), 6:QColor("red"), 7:QColor("magenta"), 8:QColor("darkYellow"), 9:QColor("blue"), 10:QColor("darkGreen"), 11:QColor("darkRed"), 12:QColor("green"), 13:QColor("darkBlue"), 14:QColor("grey"), 15:QColor("darkMagenta")} #or QColor(20,60,106)
ds.categoryColors = dColors.values() #520
ds.categoryIndices = dColors.keys() #521
ds.size = QSizeF(100.0, 100.0) #522
ds.sizeType = 0 #523 - mm(0), map units (1)
ds.labelPlacementMethod = 1 #524 - magic - from an existing example...
ds.scaleByArea = False #525
ds.minimumSize = 0 #533
ds.BackgroundColor = QColor(255,255,255,0) #Transparent White #536
ds.PenColor = QColor("black") #537
ds.penWidth = 0 #538
ds.minScaleDenominator = -1; #546
ds.maxScaleDenominator = -1; #547
#ds.diagramOrientation = 2 #551 - May only be required for histograms
#ds.barWidth = 5.0 #553
#We want a linear size interpolated version, so that Total_BA = 0 means they disappear
#564-572
dr = QgsLinearlyInterpolatedDiagramRenderer()
dr.setLowerValue( 0.0 )
dr.setLowerSize( QSizeF( 0.0, 0.0 ) )
dr.setUpperValue( 50 )
dr.setUpperSize( QSizeF(40,40) )
dr.setClassificationAttribute( 16 )
dr.setDiagram( diagram )
dr.setDiagramSettings( ds )
#And now finally put it into the layer... :-)
l.setDiagramRenderer( dr ) #572
dls = QgsDiagramLayerSettings() #575
dls.dist = 0
dls.priority = 0
dls.xPosColumn = -1 #585
dls.yPosColumn = -1
dls.placement = 0 #588
l.setDiagramLayerSettings( dls ) #593
#Refresh map
if hasattr(l, "setCacheImage"): l.setCacheImage(None)
l.triggerRepaint()