ML-EvalPro is a platform designed to advance ML model evaluation by providing a comprehensive suite of tools for automated testing and analysis for supervised machine learning tasks.
- Apache Spark
- Ollama
-
After downloading open anaconda prompt and run this command
conda install conda-forge::ollama
-
From terminal run this command
ollama pull llama3
-
You can install ML-EvalPro using pip:
pip install ml-eval-pro
This script demonstrates using MLFlow Experiment with LightGBM Regression on California Housing Dataset .
import pandas as pd
from ml_eval_pro.evaluator import Evaluator
import mlflow.xgboost
import lightgbm as lgb
from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split
# Set MLFlow tracking URI and experiment name
mlflow.set_tracking_uri("http://localhost:5000")
mlflow.set_experiment("Factory-try")
# Load the California housing dataset
california_data = fetch_california_housing()
# Convert to DataFrame
california_df = pd.DataFrame(data=california_data.data, columns=california_data.feature_names)
california_df["target"] = california_data.target
# Split data into features and target
X = california_df.drop("target", axis=1)
y = california_df["target"]
# Split data into train and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
# Add context for the dataset
dataset_context = f'this dataset contains information about housing ' \
'districts in California. This dataset is often utilized ' \
'for regression tasks, particularly for predicting the ' \
'median house value for districts based on various features.'
# Add Feature Description
features_description = {'Median House Value': 'This is the target variable, '
'representing the median house value '
'for households in the district.',
'Median Income': 'The median income of households in the district, '
'which could be a significant predictor of '
'housing prices.',
'Housing Median Age': 'The median age of houses in the district, '
'providing insights into the housing stock\'s '
'age distribution.',
'Average Rooms': 'The average number of rooms in houses within the '
'district, which can reflect the size of dwellings.',
'Average Bedrooms': 'The average number of bedrooms in houses within '
'the district, indicating the typical household size.',
'Population': 'The total population of the district, '
'which may influence housing demand and prices.',
'Average Occupancy': 'The average occupancy per household, indicating '
'the housing occupancy rate.'
'providing spatial information.',
}
# Define LightGBM parameters for regression
params = {
"objective": "regression",
"metric": "rmse",
"boosting_type": "gbdt",
"learning_rate": 0.1,
"num_leaves": 31,
"max_depth": -1,
"subsample": 0.8,
"colsample_bytree": 0.8,
"random_state": 42
}
with mlflow.start_run(run_name='LightGBM_Regression_California_Test'):
# Log parameters
mlflow.log_params(params)
# Train model
train_data = lgb.Dataset(X_train, label=y_train)
test_data = lgb.Dataset(X_test, label=y_test)
model = lgb.train(params, train_data, valid_sets=[test_data])
# Log model
model_info = mlflow.lightgbm.log_model(model, "lightgbm_regression_california_model")
# When providing testing and training datasets
evaluator = Evaluator(model_uri=model_info.model_uri,
model_type="regression", test_dataset=pd.DataFrame(X_test), test_target=pd.Series(y_test),
train_dataset=pd.DataFrame(X_train),
train_target=pd.Series(y_train), evaluation_metrics=['MAE'],
dataset_context=dataset_context,
features_description=features_description)
# Uncomment when using test dataset only
# evaluator = Evaluator(model_uri=model_info.model_uri,
# model_type="regression", test_dataset=pd.DataFrame(X_test),
# test_target=pd.Series(y_test), evaluation_metrics=['MAE'],
# dataset_context=dataset_context,
# features_description=features_description)
print(evaluator.get_evaluations())