Counting number of attributes for shapefiles within folder/sub-folders using ArcPy?

Question

Currently I am using this code to extract the "Number of fields" and "Feature Count" for the shapefiles listed in the input csv

import csv
import arcpy
from os import path
csvpath = r"D:\QAQC\List2.csv"
output = r'D:\Test\Test_2.csv'
csvdir = path.split(csvpath)[0]

with open(csvpath, "r") as csvfile:
    fcs = [r[0] for r in csv.reader(csvfile)]
    with open(output, 'wb') as f:
        for fc in fcs:
            count = len(arcpy.ListFields(fc))
            fcount = arcpy.GetCount_management(fc)
            csvwriter = csv.writer(f,delimiter=',')
            csvwriter.writerow([fc,count,fcount])

    f.close()

Along with this I would like to extract number of attributes of each shapefile. For eg; if Africa1.shp has 4 fields and 40 features, theoretically there can be 160 attribute values to Africa1.shp (but some features might not have few attributes).

So I want the output like, Africa1.shp 4 40 160

Don't know whether such a function exists in ArcPy

I want the actual number of attribute values. To explain things, i've mentioned 40 * 4 = 160 attributes

Edit:

Based on the answers below and through this Q&A Whats a Python Script/Method to use to get the number of fields with data in an attribute table?, i've almost got what i want. Two issues. 1) I don't want FID & Geometry field to get calculated. For this, ive referred to this one - https://geonet.esri.com/thread/75885 - and it works.

Only issue is couldn't able to save it to a csv file. If i remove this one piece of code

field_list = [f.name for f in arcpy.ListFields(fc) if f.type not in types and f.name.lower() not in other_geom]

and use count = arcpy.ListFields(fc), i can able to save it as csv. Don't know how to figure this out

2) Count results for fields with no null/blanks values are like, row after row. It's getting listed for all the fields. Instead how to sum the count results ?

Here is the code

import csv
import arcpy
from os import path
csvpath = r"D:\RTXQAQC\Open.csv"
csvdir = path.split(csvpath)[0]
output = r'D:\Test\Test_323.csv'
with open(csvpath, "r") as csvfile:
    fcs = [r[0] for r in csv.reader(csvfile)]
    with open(output, 'wb') as f:
        for fc in fcs:

            types = ['OID','Geometry']  
            other_geom = ['shape_area','shape_length']  

            field_list = [f.name for f in arcpy.ListFields(fc) if f.type not in types and f.name.lower() not in other_geom]      

            # Get list of fields

            count = len(field_list)
            fcount = arcpy.GetCount_management(fc)

            # Set up dictionary for the fields
            countdict = {}
            for field in field_list:
                countdict[field] = 0
            # Loop through Table and see which fields are NULL or Blank
            cur = arcpy.SearchCursor(fc)
            for row in cur:
                for field in field_list:
                    if not row.getValue(field) in [None, " ", ""]:
                        countdict[field] = countdict[field] + 1

            # Print out results - shows field name and number of non NULL/Blank fields
            for f in countdict.keys():

                print (fc,count,fcount,f,countdict[f])

            csvwriter = csv.writer(f,delimiter=',')
            csvwriter.writerow([fc,count,fcount,f,countdict[f]]

And the results are

File - No of Fields(Without FID, Geometry) - Feature count - Field name - Field vaules (excluding null/blank)

D:\\Test\\shp\\sheet41.shp, 14, 87, Y, 87
D:\\Test\\shp\\sheet41.shp, 14, 87, X, 87
D:\\Test\\shp\\sheet41.shp, 14, 87, Sample No, 87
D:\\Test\\shp\\sheet41.shp, 14, 87, Cu_Method, 72
D:\\Test\\shp\\sheet42-6.shp, 7, 333, Sample No., 333
D:\\Test\\shp\\sheet42-6.shp, 7, 333, Comments, 10
D:\\Test\\shp\\sheet42-6.shp, 7, 333, Source, 0
D:\\Test\\shp\\sheet42-6.shp, 7, 333, Shape, 333

Answer 1

As I said, not near a computer so hopefully this is ok, this is how I would go about it, I will check to see if it works in a few weeks. Essentially this stores all values for all keys in a list that is then stored in a dictionary under that key, you can then analyse each list such as counts etc.

import csv
import arcpy
from os import path
csvpath = r"D:\QAQC\Open.csv"
output = r'D:\Test\Test_32.csv'
csvdir = path.split(csvpath)[0]

with open(csvpath, "r") as csvfile:
    fcs = [r[0] for r in csv.reader(csvfile)]
with open(output, 'wb') as f:
    message = ['fc', 'fieldCount', 'rowCount', 'totUniCount', 'totCount']
    a = csv.writer(f)
    a.writerows([message])


for fc in fcs:
    # Get list of fields
    fields = [f.name for f in arcpy.ListFields(fc)]
    #get number of fields
    fieldCount = len(fields)
    #get field index in the list so you know what row to search for 
    fieldIndex = {k: v for v, k in enumerate(fields)}
    rowCount = 0
    # this is a dictionary to store all values under a key in each field
    # if the files are large or you haven't got much memory this
    # might fall over
    fieldInfo = {}
    with arcpy.da.SearchCursor(fc,fields) as cursor:
        for row in cursor:
            # count the rows
            rowCount += 1
            # get the value for each item in the row
            for key, value  in fieldIndex.iteritems():
                #add fields here that you don't want to count
                if key not in ['oid']:
                    rowValue = row[value]
                    # get the existing list of values for that field
                    # create empty list if it doesn't
                    listOfFieldData = fieldInfo.get(key,[])
                    if rowValue not in [None, " ", ""]:

                        # append the value to the list and store it under the 
                        # the appropriate field
                        listOfFieldData.append(rowValue)
                    fieldInfo[key] = listOfFieldData
    totCount =0
    totUniCount = 0
    # analyse your dictionary here
    # you can even get unique for each field as well
    for key, value in fieldInfo.iteritems():
       # print field name and length
        print key, len(value), len(set(value))
        totCount += len(value)
        totUniCount += len(set(value))
    #write it
    message =[fc, fieldCount, rowCount, totUniCount, totCount]
    with open(output, 'ab') as f:
         a = csv.writer(f)
         a.writerows([message])

Answer 2

A lot of this has been covered in Summarising all fields in attribute table using ArcPy?

I think the the Summary Statistics tool will do what you want but other approaches are mentioned in the first link. Summary Statistics will create a table with the unique counts of one or more attribute fields. You can then loop through the table and add up all the counts. Code below is not tested but will get you started. (I used the code in the answers in the question I linked to so credit to the original authors.)

import csv
import arcpy
from os import path
csvpath = r"D:\QAQC\List2.csv"
output = r'D:\Test\Test_2.csv'
csvdir = path.split(csvpath)[0]
intable = "intable"
outtable = "sumstats"
casefield = "Name"
stats = []
with open(csvpath, "r") as csvfile:
fcs = [r[0] for r in csv.reader(csvfile)]
with open(output, 'wb') as f:
    for fc in fcs:
        count = len(arcpy.ListFields(fc))
        fcount = arcpy.GetCount_management(fc)
        csvwriter = csv.writer(f,delimiter=',')
        csvwriter.writerow([fc,count,fcount])
        for field in arcpy.ListFields(fc):
            # Just find the fields that have a numeric type
            if field.type in ("Double", "Integer", "Single", "SmallInteger"):
            # Add the field name and Sum statistic type
            #    to the list of fields to summarize
            stats.append([field.name, "Sum"])
            # Correct formatting of stats [["Field1", "Sum"], ["Field2", "Sum"], ...]
         # Run the Summary Statistics tool with the stats list
         arcpy.Statistics_analysis(fc, outtable, stats, casefield)
         #Loop through the statistics and sum them
         for sumfield in arcpy.ListFields(outtable):
             mycount= sum([row[0] for row in arcpy.da.SearchCursor(fc, sumfield)])
         csvwriter.writerow([fc,count,mycount])
f.close()

Answer 3

The following code reads each shapefile, counts the features and fields, counts the unique values per field, and outputs them to a csv file. It assumes List2.csv has a column headed Shapefilename where it gets the list of shapefiles.

import arcpy, csv, os

csvpath = r"D:\QAQC\List2.csv"
output = r'D:\Test\Test_2.csv'

csvdir = os.path.split(csvpath)[0]

f = open(output, 'wb')
csvwriter = csv.writer(f, delimiter=',')
csvwriter.writerow(['Shapefile', 'Feature Count', 'Field Count', 'Possible Values', 'Unique Values'])

shpList = list()

with arcpy.da.SearchCursor(csvpath, 'Shapefilename') as cursor:
    for row in cursor:
        shpList.append(row[0])

for shpFile in shpList:
    shpPath = os.path.join(csvdir, shpFile)
    featCount = int(arcpy.GetCount_management(shpPath).getOutput(0))
    fieldList = arcpy.ListFields(shpPath)
    fieldCount = len(fieldList)

    valCount = 0

    if fieldCount > 0:
        for field in fieldList:
            uniqueValues = set()
            with arcpy.da.SearchCursor(shpPath, [field.name]) as cursor:
                for row in cursor:
                    if row[0] not in uniqueValues:
                        uniqueValues.add(row[0])
            valCount += len(uniqueValues)

        print "Shapefile = {}, Features = {}, Fields = {}, Possible Values = {}, Unique Values = {}".format(shpFile, featCount, fieldCount, featCount * fieldCount, valCount)

        csvwriter.writerow([shpFile, featCount, fieldCount, featCount * fieldCount, valCount])

f.close()