kustopy is a Python SDK for Azure Data Explorer (Kusto).
Ingestion using pyspark dataframes, is quite slow (compared to ingestion using pandas dataframes). But data is immediately in database after the command is run, while for pandas dataframes it can take while for data to appear.
pip install kustopy
from kustopy import KustoPyClient as kpc
# Set up the query client
client = kpc.Client(cluster='https://sample.kusto.windows.net/',
database='sample',
client_id=client_id,
client_secret=client_secret,
tenant_id=tenant_id,
truncation=False)
# Get list of all tables available in database
client.get_table_names()
# Get list of all tables available in database
client.get_schema(table_name='SampleTable')
# Write standard kusto queries. To get the response as pandas dataframe set dataframe to True.
client.query(user_input='SampleTable | take 100 | where fruit=="apple"', data_frame=True)
# Drop table from the database
client.drop_table(table_name='SampleTable')
# Drop table from the database
client.drop_dublicates(table_name='SampleTable')
# Write pandas or pyspark dataframe to the database. If the table exists you will get an Error.
client.write_table(data_frame=df, table_name='SampleTable', folder='Sample')
# Write pandas or pyspark dataframe to the database. If the table exists it will be replaced, otherwise created.
client.write_replace_table(data_frame=df, table_name='SampleTable', folder='Sample')
# Write pandas or pyspark dataframe to the database. If the table exists it will be expanded, otherwise created.
client.write_append_table(data_frame=df, table_name='SampleTable', folder='Sample')