🌧️ prettify_exec_info
Introduction
Function that gives info about cost and time about the scraper call
Implementation
"""
Prettify the execution information of the graph.
"""
import pandas as pd
def prettify_exec_info(complete_result: dict) -> pd.DataFrame:
"""
Transform the execution information of the graph into a DataFrame for better visualization.
Args:
- complete_result (dict): The complete execution information of the graph.
Returns:
- pd.DataFrame: The execution information of the graph in a DataFrame.
"""
nodes_info = complete_result['nodes_info']
total_info = {
'total_exec_time': complete_result['total_exec_time'],
'total_model_info': complete_result['total_model_info']
}
# Convert node-specific information to DataFrame
flat_data = []
for node_name, node_info in nodes_info.items():
flat_data.append({
'Node': node_name,
'Execution Time': node_info['exec_time'],
# Unpack the model_info dict into the row
**node_info['model_info']
})
df_nodes = pd.DataFrame(flat_data)
# Add a row for the total execution time and total model info
total_row = {
'Node': 'Total',
'Execution Time': total_info['total_exec_time'],
# Unpack the total_model_info dict into the row
**total_info['total_model_info']
}
df_total = pd.DataFrame([total_row])
# Combine the nodes DataFrame with the total info DataFrame
df_combined_with_total = pd.concat([df_nodes, df_total], ignore_index=True)
return df_combined_with_total
Example
from scrapegraphai.graphs import SmartScraperGraph
from scrapegraphai.utils import prettify_exec_info
graph_config = {
"llm": {
"model": "ollama/mistral",
"temperature": 1,
"format": "json", # Ollama needs the format to be specified explicitly
"base_url": "http://localhost:11434", # set ollama URL arbitrarily
},
"embeddings": {
"model": "ollama/nomic-embed-text",
"temperature": 0,
"base_url": "http://localhost:11434", # set ollama URL arbitrarily
}
}
smart_scraper_graph = SmartScraperGraph(
prompt="List me all the news with their description.",
source="https://perinim.github.io/projects",
config=graph_config
)
result = smart_scraper_graph.run()
print(result)
graph_exec_info = smart_scraper_graph.get_execution_info()
print(prettify_exec_info(graph_exec_info))