Set a workspace inside a multiprocessing function inside a toolbox in ArcGIS Pro

Question

I am trying to set a workspace inside a multiprocessing function of an ArcGIS Pro toolbox. It works perfectly in standalone script, but when I use it in a toolbox I get this error message:

  multiprocessing.pool.RemoteTraceback:  """ Traceback (most recent call last):   File "C:\Program
 Files\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\Lib\multiprocessing\pool.py",
 line 121, in worker
     result = (True, func(*args, **kwds))   File "C:\Program Files\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\Lib\multiprocessing\pool.py",
 line 44, in mapstar
     return list(map(*args))   File "G:\OutilsProdDIF\outils\Transfert_donnees_compilees_decoupes\source\transfert_donnees_compilees_decoupees.py",
 line 253, in multiprocessing3
     gdb_number = get_gdb_number(gdb)   File "G:\OutilsProdDIF\outils\Transfert_donnees_compilees_decoupes\source\transfert_donnees_compilees_decoupees.py",
 line 31, in get_gdb_number
     for feature in list_feature: TypeError: 'NoneType' object is not iterable """

The above exception was the direct cause of the following exception:

 Traceback (most recent call last):   File
 "G:\OutilsProdDIF\outils\Transfert_donnees_compilees_decoupes\source\transfert_donnees_compilees_decoupees.py",
 line 450, in <module>
     pool.map(transfert_donnees_compilees_decoupees.multiprocessing3, list_gdb_with_meta_cmp)   File "C:\Program
 Files\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\Lib\multiprocessing\pool.py",
 line 268, in map
     return self._map_async(func, iterable, mapstar, chunksize).get()   File "C:\Program
 Files\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\Lib\multiprocessing\pool.py",
 line 657, in get
     raise self._value TypeError: 'NoneType' object is not iterable  Échec de l’exécution de (transfertdonneescompileesdecoupees).

It's like the arcpy.env.workspace doesn't work because the result of the ListFeatures() is None. I really don't understand why it doesn't work inside toolbox.

My script:

import os
import multiprocessing
import sys
import fnmatch
import arcpy
import csv
import sqlite3
import shutil
try:
    #permet d'ajouter le package pandas à l'utilisateur s'il ne l'a pas
    destination = r"C:\Program Files\ArcGIS\Pro\bin\Python\envs\arcgispro-py3\Lib\site-packages"
    folder = r"G:\OutilsProdDIF\outils\Transfert_donnees_compilees_decoupes\prerequis\pandas"
    shutil.copytree(folder, destination)
    folder = r"G:\OutilsProdDIF\outils\Transfert_donnees_compilees_decoupes\prerequis\pandas-0.24.2.dist-info"
    shutil.copytree(folder, destination)
except:
    pass
import pandas
from class_convertIEQM import ConvertIEQM


def get_gdb_number():
    list_feature = arcpy.ListFeatureClasses()
    for feature in list_feature:
        if feature.startswith('META_ORI'):
            meta_ori = feature

    name_split = meta_ori.split('_')
    gdb_number = name_split[-1]
    return gdb_number


def find_dirs(pattern, path):
    result = []
    for root, dirs, files in os.walk(path):
        for name in dirs:
            if fnmatch.fnmatch(name, pattern):
                result.append(os.path.join(root, name))
    return result


def find_files(pattern, path):
    result = []
    for root, dirs, files in os.walk(path):
        for name in files:
            if fnmatch.fnmatch(name, pattern):
                result.append(os.path.join(root, name))
    return result


def create_logiciels_temp_if_not_exist():
    try:
        os.makedirs(r'C:\Logiciels\temp')
    except:
        pass


def copy_directory(source, destination):
    for root, dirs, files in os.walk(source):
        if not os.path.isdir(root):
            os.makedirs(root)

        for file in files:
            rel_path = root.replace(source, '').lstrip(os.sep)
            dest_path = os.path.join(destination, rel_path)

            if not os.path.isdir(dest_path):
                os.makedirs(dest_path)

            shutil.copyfile(os.path.join(root, file), os.path.join(dest_path, file))


def unique_values(table, field):
    list_unique_values = []
    list_unique_values.clear()
    cursor = arcpy.SearchCursor(table)
    for row in cursor:
        if row.getValue(field) in list_unique_values:
            pass
        else:
            list_unique_values.append(row.getValue(field))
    return list_unique_values


def convert_csv_to_sqlite(csv_file, sqlite_file, table_name, field_type=None):
    if os.path.exists(sqlite_file):
        os.remove(sqlite_file)

    con = sqlite3.connect(sqlite_file)
    df = pandas.read_csv(csv_file, dtype = field_type)
    df.to_sql(table_name, con, if_exists='append', index=False)


def convert_sqlite_to_csv(sqlite_file, sql_query, csv_output, list_field):
    con = sqlite3.connect(sqlite_file)
    cur = con.cursor()
    data = cur.execute(sql_query)

    try:
        os.remove(csv_output)
    except:
        pass

    file = open(csv_output, 'w')
    writer = csv.writer(file)
    writer.writerow(list_field)
    writer.writerows(data)


def create_index_sqlite(sqlite_file, index_name, table_name, field_name):
    con = sqlite3.connect(sqlite_file)
    cur = con.cursor()
    index = f"CREATE INDEX {index_name} ON {table_name}({field_name})"
    cur.execute(index)
    con.commit()
    con.close()


def convert_list_for_sql_query(list):
    elements = (', '.join("'{0}'".format(x) for x in list))
    sql_list = f"({elements})"
    return sql_list


def multiprocessing1(gdb):
    meta_cmp = gdb[1]
    gdb = gdb[0]

    arcpy.env.workspace = gdb
    gdb_number = get_gdb_number()

    #VALIDATION (Si les tables existes déjà)
    create_dendro_pee = True
    create_meta_cmp = True

    list_table = arcpy.ListTables()
    for table in list_table:
        if table == f"DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}":
            print(f"DENDRO_PEE_TIGES_DHP_ORI_{gdb_number} existe déjà!")
            create_dendro_pee = False
        if table == f"META_CMP_ORI_{gdb_number}":
            print(f"META_CMP_ORI_{gdb_number} existe déjà!")
            create_meta_cmp = False

    #DENDO_PEE_TIGES_DHP_ORI
    if create_dendro_pee:
        list_geocode = unique_values(f'META_ORI_{gdb_number}', 'GEOCODE')
        sql_query = f"SELECT * FROM DENDRO_PEE WHERE GEOCODE IN {convert_list_for_sql_query(list_geocode)}"
        csv_output = fr"C:\Logiciels\temp\transfert_donnees_compilees_decoupees\OUTPUT_DENDO_PEE_TIGES_DHP_ORI_{gdb_number}.csv"
        list_field = ['GEOCODE', 'CO_CMP', 'CL_DHP', 'TIGE_HA', 'ST_HA', 'VMB_HA']

        convert_sqlite_to_csv(r"C:\Logiciels\temp\transfert_donnees_compilees_decoupees\DENDRO.db", sql_query, csv_output, list_field)

        template = r"C:\Logiciels\temp\transfert_donnees_compilees_decoupees\GDB_BASE_DENDRO_PEE_TIGES_DHP_ORI.gdb\DENDRO_PEE_TIGES_DHP_ORI"
        arcpy.CreateFileGDB_management(r"C:\Logiciels\temp\transfert_donnees_compilees_decoupees", f"GDB_BASE_DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}.gdb")
        arcpy.CreateTable_management(fr"C:\Logiciels\temp\transfert_donnees_compilees_decoupees\GDB_BASE_DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}.gdb", "DENDRO_PEE_TIGES_DHP_ORI", template)

        dendro_pee_table = fr"C:\Logiciels\temp\transfert_donnees_compilees_decoupees\GDB_BASE_DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}.gdb\DENDRO_PEE_TIGES_DHP_ORI"

        arcpy.env.workspace = gdb
        arcpy.DeleteRows_management(dendro_pee_table)
        arcpy.Append_management(csv_output, dendro_pee_table, "NO_TEST")

        arcpy.TableToTable_conversion(dendro_pee_table, gdb, f"DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}")

        arcpy.AddIndex_management(f"DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}", ["GEOCODE"], "GEOCODE")

        origin_table = f"DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}"
        destination_table = f"PEE_ORI_{gdb_number}"
        arcpy.CreateRelationshipClass_management(origin_table, destination_table, "CMP_TIGES_DHP", "SIMPLE", destination_table, origin_table, "NONE", "ONE_TO_MANY", "NONE", "GEOCODE", "GEOCODE", '', '')

        arcpy.Delete_management(fr"C:\Logiciels\temp\transfert_donnees_compilees_decoupees\GDB_BASE_DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}.gdb")
        arcpy.Delete_management(csv_output)

    #META_CMP_ORI
    if create_meta_cmp:
        list_no_uco = unique_values(f'META_ORI_{gdb_number}', 'NO_UCO')
        inTable = meta_cmp
        outLocation = gdb
        outTable = f"META_CMP_ORI_{gdb_number}"
        expression = f"NO_UCO IN {convert_list_for_sql_query(list_no_uco)}"
        arcpy.TableToTable_conversion(inTable, outLocation, outTable, expression)

    #Convertir en gpkg
    gdb_name = os.path.basename(gdb)
    gdb_name = gdb_name[0:-7]

    objConvertIeqm = ConvertIEQM(gdbAConvert=gdb,
                                 gpkgNomSortie=f"{gdb_name}.gpkg",
                                 pathGpkgStyle="G:/OutilsProdDIF/modules_communs/python27/conversionFormat/prerequis/ieqm_styles.gpkg",
                                 pathGpkgCode="",
                                 pathGdal="G:/OutilsProdDIF/modules_communs/gdal/gdal2.3.2",
                                 # pathGdal="",
                                 nbCaractereAEnleverNomTab=4, correctionGeom=False)
    objConvertIeqm.conversion()


def multiprocessing2(gdb):
    import arcpy
    arcpy.env.workspace = gdb
    gdb_number = get_gdb_number()
    gdb_10 = gdb[0:-6] + '10.gdb'

    #VALIDATION (Si les tables existes déjà)
    create_dendro_pee = True
    create_meta_cmp = True

    list_table = arcpy.ListTables()
    for table in list_table:
        if table == f"DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}":
            print(f"DENDRO_PEE_TIGES_DHP_ORI_{gdb_number} existe déjà!")
            create_dendro_pee = False
        if table == f"META_CMP_ORI_{gdb_number}":
            print(f"META_CMP_ORI_{gdb_number} existe déjà!")
            create_meta_cmp = False


    #DENDRO_PEE_TIGES_DHP_ORI
    if create_dendro_pee:
        inTable = gdb_10 + fr"\\DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}"
        outLocation = gdb
        outTable = f"DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}"
        arcpy.TableToTable_conversion(inTable, outLocation, outTable)
        arcpy.AddIndex_management(f"DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}", ["GEOCODE"], "GEOCODE")

        origin_table = f"DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}"
        destination_table = f"PEE_ORI_{gdb_number}"
        arcpy.CreateRelationshipClass_management(origin_table, destination_table, "CMP_TIGES_DHP", "SIMPLE", destination_table, origin_table, "NONE", "ONE_TO_MANY", "NONE", "GEOCODE", "GEOCODE", '', '')

    #META_CMP_ORI
    if create_meta_cmp:
        inTable = gdb_10 + fr"\\META_CMP_ORI_{gdb_number}"
        outLocation = gdb
        outTable = f"META_CMP_ORI_{gdb_number}"
        arcpy.TableToTable_conversion(inTable, outLocation, outTable)


def multiprocessing3(gdb):
    meta_cmp = gdb[1]
    gdb = gdb[0]

    arcpy.env.Workspace = gdb



    gdb_number = get_gdb_number()

    #VALIDATION (Si les tables existes déjà)
    create_dendro_pee = True
    create_meta_cmp = True

    list_table = arcpy.ListTables()
    for table in list_table:
        if table == f"DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}":
            print(f"DENDRO_PEE_TIGES_DHP_ORI_{gdb_number} existe déjà!")
            create_dendro_pee = False
        if table == f"META_CMP_ORI_{gdb_number}":
            print(f"META_CMP_ORI_{gdb_number} existe déjà!")
            create_meta_cmp = False

    #DENDO_PEE_TIGES_DHP_ORI
    if create_dendro_pee:
        list_geocode = unique_values(f'META_ORI_{gdb_number}', 'GEOCODE')
        sql_query = f"SELECT * FROM DENDRO_PEE WHERE GEOCODE IN {convert_list_for_sql_query(list_geocode)}"
        csv_output = fr"C:\Logiciels\temp\transfert_donnees_compilees_decoupees\OUTPUT_DENDO_PEE_TIGES_DHP_ORI_{gdb_number}.csv"
        list_field = ['GEOCODE', 'CO_CMP', 'CL_DHP', 'TIGE_HA', 'ST_HA', 'VMB_HA']

        convert_sqlite_to_csv(r"C:\Logiciels\temp\transfert_donnees_compilees_decoupees\DENDRO.db", sql_query, csv_output, list_field)

        template = r"C:\Logiciels/temp/transfert_donnees_compilees_decoupees/GDB_BASE_DENDRO_PEE_TIGES_DHP_ORI.gdb/DENDRO_PEE_TIGES_DHP_ORI"
        arcpy.CreateFileGDB_management(r"C:\Logiciels\temp\transfert_donnees_compilees_decoupees", f"GDB_BASE_DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}.gdb")
        arcpy.CreateTable_management(fr"C:\Logiciels\temp\transfert_donnees_compilees_decoupees\GDB_BASE_DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}.gdb", "DENDRO_PEE_TIGES_DHP_ORI", template)

        dendro_pee_table = fr"C:\Logiciels\temp\transfert_donnees_compilees_decoupees\GDB_BASE_DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}.gdb\DENDRO_PEE_TIGES_DHP_ORI"

        arcpy.env.workspace = gdb
        arcpy.DeleteRows_management(dendro_pee_table)
        arcpy.Append_management(csv_output, dendro_pee_table, "NO_TEST")

        arcpy.TableToTable_conversion(dendro_pee_table, gdb, f"DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}")

        arcpy.AddIndex_management(f"DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}", ["GEOCODE"], "GEOCODE")

        origin_table = f"DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}"
        destination_table = f"PEE_ORI_{gdb_number}"
        arcpy.CreateRelationshipClass_management(origin_table, destination_table, "CMP_TIGES_DHP", "SIMPLE", destination_table, origin_table, "NONE", "ONE_TO_MANY", "NONE", "GEOCODE", "GEOCODE", '', '')

        arcpy.Delete_management(fr"C:\Logiciels\temp\transfert_donnees_compilees_decoupees\GDB_BASE_DENDRO_PEE_TIGES_DHP_ORI_{gdb_number}.gdb")
        arcpy.Delete_management(csv_output)

    #META_CMP_ORI
    if create_meta_cmp:
        list_no_uco = unique_values(f'META_ORI_{gdb_number}', 'NO_UCO')
        inTable = meta_cmp
        outLocation = gdb
        outTable = f"META_CMP_ORI_{gdb_number}"
        expression = f"NO_UCO IN {convert_list_for_sql_query(list_no_uco)}"
        arcpy.TableToTable_conversion(inTable, outLocation, outTable, expression)


    #Convertir en gpkg
    gdb_name = os.path.basename(gdb)

    if "ORI" in gdb_name:
        if "TFR" in gdb_name:
            number_carac = 7
        elif 'UA' in gdb_name:
            number_carac = 6
        else:
            # agence
            number_carac = 4
    else:
        pass

    gdb_name = gdb_name.split('.')[0]

    objConvertIeqm = ConvertIEQM(gdbAConvert=gdb,
                                 gpkgNomSortie=f"{gdb_name}.gpkg",
                                 pathGpkgStyle="G:/OutilsProdDIF/modules_communs/python27/conversionFormat/prerequis/ieqm_styles.gpkg",
                                 pathGpkgCode="",
                                 pathGdal="G:/OutilsProdDIF/modules_communs/gdal/gdal2.3.2",
                                 # pathGdal="",
                                 nbCaractereAEnleverNomTab=f'{number_carac}', correctionGeom=False)
    objConvertIeqm.conversion()







if __name__ == "__main__":

    sys.path.append(r"G:\OutilsProdDIF\outils\Transfert_donnees_compilees_decoupes\source")
    import transfert_donnees_compilees_decoupees

    arcpy.SetLogHistory(False) # permet d'ameliorer les temps de traitement
    multiprocessing.set_executable(os.path.join(sys.exec_prefix, 'pythonw.exe')) # permet le multiprocessing dans un script arc tool box sans passer par un bat file

    arcpy.env.overwriteOutput = True

    create_logiciels_temp_if_not_exist()

    source = r"G:\OutilsProdDIF\outils\Transfert_donnees_compilees_decoupes\prerequis"
    destination = r"C:\Logiciels\temp\transfert_donnees_compilees_decoupees"
    csv_file = arcpy.GetParameterAsText(1)
    sqlite_file = destination + r"\DENDRO.db"
    table_name = "DENDRO_PEE"
    field_type = {"GEOCODE": "str", "CO_CMP": "str", "CL_DHP": "str", "TIGE_HA": "str", "ST_HA": "str", "VMB_HA": "str"}

    #base de données sqlite à partir du csv
    create_sqlite_table = arcpy.GetParameter(0)
    if create_sqlite_table:
        copy_directory(source, destination) #permet de copier les fichiers prérequis au programme et de créer un répertoire dans le C:\Logiciels\temp\
        convert_csv_to_sqlite(csv_file, sqlite_file, table_name, field_type)
        create_index_sqlite(sqlite_file, 'index_geocode', table_name, 'GEOCODE')

    #permet de vider les gdb et csv s'il y a eu un bogue et qui ne se sont pas supprimés la dernière fois que l'outil a été lancé
    list_gdb_to_del = find_dirs('*.gdb', destination)
    list_csv_to_del = find_files('*.csv', destination)
    for gdb in list_gdb_to_del:
        if 'GDB_BASE_DENDRO_PEE_TIGES_DHP_ORI.gdb' in gdb:
            pass
        else:
            arcpy.Delete_management(gdb)
    for csv in list_csv_to_del:
        arcpy.Delete_management(csv)

    #liste des gdb
    gdb_directory = arcpy.GetParameterAsText(2)
    list_gdb = find_dirs('*.gdb', gdb_directory) #permet d'avoir une liste avec les chemins de toutes les gdb dans le répertoire

    #Permet d'enlever les gdb version 9 de la liste pour éviter de faire les calculs en double
    #Les tables seront copier de la version 10 à la fin
    list_gdb_10 = find_dirs('*_10.gdb', gdb_directory)
    list_gdb_93 = find_dirs('*_93.gdb', gdb_directory)

    #permet de différencier les deux types de répertoires GDB (feuillet 250k ou par ua, agence et tfr)
    feuillet_250k = False

    #creer shcema_ini
    #permet d'ouvrir un fichier csv dans arcgis avec les bons types de champs
    fichier = open(r"C:\Logiciels\temp\transfert_donnees_compilees_decoupees\schema.ini", "a")
    fichier.truncate(0)

    if feuillet_250k:
        list_ini = list_gdb_10
    else:
        list_ini = list_gdb
    for gdb in list_ini:
        arcpy.env.workspace = gdb
        gdb_number = get_gdb_number()
        fichier.write(f"""[OUTPUT_DENDO_PEE_TIGES_DHP_ORI_{gdb_number}.csv]
Col1=GEOCODE Text
Col2=CO_CMP Text
Col3=CL_DHP Text
Col4=TIGE_HA Double
Col5=ST_HA Double
Col6=VMB_HA Double

""")
    fichier.close()


    #détection automatique  pour savoir si c'est des gdb feuillet_250k ou gdb par ua, agence et tfr
    if len(list_gdb_93)>=1:
        feuillet_250k = True

    #traitement multiprocessing
    meta_cmp = arcpy.GetParameterAsText(3)
    if feuillet_250k:
        list_gdb_with_meta_cmp = []

        for gdb in list_gdb_10:
            list_gdb_with_meta_cmp.append([gdb, meta_cmp])

        #multiprocessing
        core_nb = multiprocessing.cpu_count()
        pool = multiprocessing.Pool(processes=core_nb - 1)
        pool.map(transfert_donnees_compilees_decoupees.multiprocessing1, list_gdb_with_meta_cmp)
        pool.close()
        pool.join()

        core_nb = multiprocessing.cpu_count()
        pool = multiprocessing.Pool(processes=core_nb - 1)
        pool.map(transfert_donnees_compilees_decoupees.multiprocessing2, list_gdb_93)
        pool.close()
        pool.join()

        del pool
    else:
        #pour répertoire gdb par ua, agence et tfr

        #permet d'avoir meta_cmp à l'intérieur du multiprocessing
        list_gdb_with_meta_cmp = []

        for gdb in list_gdb:
            list_gdb_with_meta_cmp.append([gdb, meta_cmp])

        #multiprocessing
        core_nb = multiprocessing.cpu_count()
        pool = multiprocessing.Pool(processes=core_nb - 1)
        pool.map(transfert_donnees_compilees_decoupees.multiprocessing3, list_gdb_with_meta_cmp)
        pool.close()
        pool.join()

Might be more of a workaround than a solution, but if it works standalone, have you tried checking that box in the script's properties to run it out of process? You could also conceivably write a little wrapper script that launches the real script in a subprocess, which might be essentially the same thing but could give you more control over it. — mikewatt, Commented May 5, 2021 at 19:29
yes, I can lauch to program with a bat file that lauch the script instead of the python file directly in the arctool box. But with this method, I don't see error messages if the script bugs for different raisons... — Philippe Leblanc, Commented May 10, 2021 at 19:30

Stack Exchange Network

Set a workspace inside a multiprocessing function inside a toolbox in ArcGIS Pro

0

Your Answer

Browse other questions tagged
arcpy
parallel-processing
arcgis-platform
workspace
toolbox
or ask your own question.

Hot Network Questions

Set a workspace inside a multiprocessing function inside a toolbox in ArcGIS Pro

0

Know someone who can answer? Share a link to this question via email, Twitter, or Facebook.

Your Answer

Sign up or log in

Post as a guest

Browse other questions tagged arcpyparallel-processingarcgis-platformworkspacetoolbox or ask your own question.

Related

Hot Network Questions

Browse other questions tagged
arcpy
parallel-processing
arcgis-platform
workspace
toolbox
or ask your own question.