The correct approach is to directly modify the ValueRelationDataValueRelationData
object you get from vlayer.valueRelation( idx )
vlayer.setEditType(12, QgsVectorLayer.ValueRelation)
vlayer.valueRelation(12).mLayer = 'releatedtable20131027113855772'
vlayer.valueRelation(12).mKey = 'key'
vlayer.valueRelation(12).mValue = 'value'
vlayer.valueRelation(12).mAllowNull = False
vlayer.valueRelation(12).mOrderByValue = True
# vlayer.valueRelation(12).mAllowMulti = True
# vlayer.valueRelation(12).mFilterExpression = 'COALESCE( "fieldA", \'<NULL>\' )'