AWS SageMaker for Beginners: Complete Setup Guide with Cost Optimization

# For experimentation and learning
instance_type = 'ml.t3.medium'    # ~$0.05/hour

# For small datasets and simple models
instance_type = 'ml.t3.large'     # ~$0.10/hour

# For GPU workloads (expensive!)
instance_type = 'ml.p3.2xlarge'   # ~$3.00/hour

training_job_config = {
    'TrainingJobName': 'my-training-job',
    'AlgorithmSpecification': {
        'TrainingImage': 'your-algorithm-image',
        'TrainingInputMode': 'File'
    },
    'RoleArn': 'your-sagemaker-role-arn',
    'InputDataConfig': [...],
    'OutputDataConfig': {...},
    'ResourceConfig': {
        'InstanceType': 'ml.p3.2xlarge',
        'InstanceCount': 1,
        'UseSpotInstances': True,
        'MaxWaitTimeInSeconds': 3600,  # 1 hour
        'MaxRuntimeInSeconds': 3600
    },
    'StoppingCondition': {
        'MaxRuntimeInSeconds': 3600
    }
}

# Lifecycle configuration script
lifecycle_config_script = """
#!/bin/bash
set -e

# Auto-shutdown after 2 hours of inactivity
IDLE_TIME=7200

echo "Starting idle check script"

while true; do
    # Check if Jupyter is running and get last activity
    if pgrep -f jupyter > /dev/null; then
        LAST_ACTIVITY=$(stat -c %Y /home/ec2-user/.jupyter/lab/workspaces/default-*.jupyterlab-workspace)
        CURRENT_TIME=$(date +%s)
        TIME_DIFF=$((CURRENT_TIME - LAST_ACTIVITY))

        if [ $TIME_DIFF -gt $IDLE_TIME ]; then
            echo "Shutting down due to inactivity"
            shutdown -h now
        fi
    fi
    sleep 300  # Check every 5 minutes
done
"""

import boto3
from datetime import datetime, timedelta

def get_sagemaker_costs():
    client = boto3.client('ce', region_name='us-east-1')

    # Get costs for last 30 days
    end_date = datetime.now().date()
    start_date = end_date - timedelta(days=30)

    response = client.get_cost_and_usage(
        TimePeriod={
            'Start': start_date.strftime('%Y-%m-%d'),
            'End': end_date.strftime('%Y-%m-%d')
        },
        Granularity='DAILY',
        Metrics=['BlendedCost'],
        Filter={
            'Dimensions': {
                'Key': 'SERVICE',
                'Values': ['Amazon SageMaker']
            }
        }
    )

    total_cost = 0
    for result in response['ResultsByTime']:
        cost = float(result['Groups'][0]['Metrics']['BlendedCost']['Amount'])
        total_cost += cost

    return total_cost

# Check costs and alert if over budget
total_cost = get_sagemaker_costs()
budget_limit = 50.0

if total_cost > budget_limit:
    print(f"ALERT: SageMaker costs (${total_cost:.2f}) exceeded budget (${budget_limit:.2f})")
    # Send notification (integrate with SNS, email, etc.)

import boto3

def cleanup_sagemaker_resources():
    sagemaker = boto3.client('sagemaker')

    # Delete unused notebook instances
    notebooks = sagemaker.list_notebook_instances()
    for notebook in notebooks['NotebookInstances']:
        if notebook['NotebookInstanceStatus'] == 'Stopped':
            # Check if stopped for more than 7 days
            stopped_time = notebook['LastModifiedTime']
            days_stopped = (datetime.now() - stopped_time.replace(tzinfo=None)).days

            if days_stopped > 7:
                print(f"Deleting old notebook: {notebook['NotebookInstanceName']}")
                sagemaker.delete_notebook_instance(
                    NotebookInstanceName=notebook['NotebookInstanceName']
                )

    # Delete unused endpoints
    endpoints = sagemaker.list_endpoints()
    for endpoint in endpoints['Endpoints']:
        if endpoint['EndpointStatus'] == 'OutOfService':
            print(f"Deleting out-of-service endpoint: {endpoint['EndpointName']}")
            sagemaker.delete_endpoint(EndpointName=endpoint['EndpointName'])

if __name__ == "__main__":
    cleanup_sagemaker_resources()

# Essential ML libraries
import pandas as pd  # Data manipulation and analysis library
import numpy as np   # Numerical computing library
import matplotlib.pyplot as plt  # Basic plotting library
import seaborn as sns  # Statistical data visualization library
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error, r2_score

# Set plotting style
plt.style.use('seaborn-v0_8')
sns.set_palette("husl")

print("Libraries imported successfully!")

# Create realistic housing data
np.random.seed(42)
n_samples = 1000

# Generate features
house_data = {
    'size_sqft': np.random.normal(2000, 500, n_samples),
    'bedrooms': np.random.randint(1, 6, n_samples),
    'bathrooms': np.random.randint(1, 4, n_samples),
    'age_years': np.random.randint(0, 50, n_samples),
    'lot_size': np.random.normal(8000, 2000, n_samples),
    'garage_spaces': np.random.randint(0, 3, n_samples)
}

# Create target variable (price)
base_price = 300000
df = pd.DataFrame(house_data)

# Add realistic price relationships
df['price'] = (
    base_price +
    (df['size_sqft'] - 2000) * 150 +           # Size impact
    df['bedrooms'] * 25000 +                   # Bedroom impact
    df['bathrooms'] * 15000 +                  # Bathroom impact
    df['lot_size'] * 10 +                      # Lot size impact
    df['garage_spaces'] * 10000 -              # Garage impact
    df['age_years'] * 3000                     # Age depreciation
)

# Add some noise and ensure positive prices
df['price'] += np.random.normal(0, 25000, n_samples)
df['price'] = df['price'].clip(lower=75000)

print(f"Generated {len(df)} house records")
print(df.head())

# Basic statistics
print("Dataset Overview:")
print(df.describe())

# Correlation analysis
correlation_matrix = df.corr()
print("\nFeature Correlations with Price:")
print(correlation_matrix['price'].sort_values(ascending=False))

# Visualize relationships
fig, axes = plt.subplots(2, 3, figsize=(15, 10))

# Scatter plots
axes[0, 0].scatter(df['size_sqft'], df['price'], alpha=0.6)
axes[0, 0].set_xlabel('Size (sq ft)')
axes[0, 0].set_ylabel('Price ($)')
axes[0, 0].set_title('Size vs Price')

axes[0, 1].scatter(df['bedrooms'], df['price'], alpha=0.6)
axes[0, 1].set_xlabel('Bedrooms')
axes[0, 1].set_ylabel('Price ($)')
axes[0, 1].set_title('Bedrooms vs Price')

axes[0, 2].scatter(df['age_years'], df['price'], alpha=0.6)
axes[0, 2].set_xlabel('Age (years)')
axes[0, 2].set_ylabel('Price ($)')
axes[0, 2].set_title('Age vs Price')

# Box plots
sns.boxplot(x='bathrooms', y='price', data=df, ax=axes[1, 0])
axes[1, 0].set_title('Bathrooms vs Price')

sns.boxplot(x='garage_spaces', y='price', data=df, ax=axes[1, 1])
axes[1, 1].set_title('Garage Spaces vs Price')

# Histogram
axes[1, 2].hist(df['price'], bins=30, alpha=0.7, edgecolor='black')
axes[1, 2].set_xlabel('Price ($)')
axes[1, 2].set_ylabel('Frequency')
axes[1, 2].set_title('Price Distribution')

plt.tight_layout()
plt.show()

# Prepare features and target
features = ['size_sqft', 'bedrooms', 'bathrooms', 'age_years', 'lot_size', 'garage_spaces']
X = df[features]
y = df['price']

# Split data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

print(f"Training set: {X_train.shape[0]} samples")
print(f"Testing set: {X_test.shape[0]} samples")

# Create and train the model
model = LinearRegression()
model.fit(X_train, y_train)

print("Model trained successfully!")

# Make predictions
y_pred = model.predict(X_test)

# Evaluate model performance
mse = mean_squared_error(y_test, y_pred)
rmse = np.sqrt(mse)
r2 = r2_score(y_test, y_pred)

print(f"Model Performance:")
print(f"Root Mean Squared Error: ${rmse:,.0f}")
print(f"R² Score: {r2:.3f}")

# Feature importance
feature_importance = pd.DataFrame({
    'feature': features,
    'importance': model.coef_
})
feature_importance = feature_importance.sort_values('importance', ascending=False)

print("\nFeature Importance:")
for _, row in feature_importance.iterrows():
    print(f"{row['feature']}: ${row['importance']:,.0f} impact per unit")

# Predict price for a new house
new_house = pd.DataFrame({
    'size_sqft': [2500],
    'bedrooms': [3],
    'bathrooms': [2],
    'age_years': [5],
    'lot_size': [8500],
    'garage_spaces': [2]
})

predicted_price = model.predict(new_house)[0]
confidence_range = rmse * 1.96  # 95% confidence interval

print(f"🏠 House Details:")
print(f"   Size: {new_house['size_sqft'][0]} sq ft")
print(f"   Bedrooms: {new_house['bedrooms'][0]}")
print(f"   Bathrooms: {new_house['bathrooms'][0]}")
print(f"   Age: {new_house['age_years'][0]} years")
print(f"   Lot Size: {new_house['lot_size'][0]} sq ft")
print(f"   Garage Spaces: {new_house['garage_spaces'][0]}")

print(f"\n💰 Predicted Price: ${predicted_price:,.0f}")
print(f"95% Confidence Range: ${predicted_price - confidence_range:,.0f} - ${predicted_price + confidence_range:,.0f}")

import sagemaker
from sagemaker.automl import AutoML

# Initialize Autopilot job
auto_ml = AutoML(
    role=sagemaker.get_execution_role(),
    target_attribute_name='price',
    problem_type='regression',
    max_candidates=50
)

# Launch AutoML job
auto_ml.fit(
    inputs=sagemaker.inputs.TrainingInput(
        s3_data_location,
        content_type='text/csv'
    ),
    job_name='house-price-automl'
)

from sagemaker.workflow.pipeline import Pipeline
from sagemaker.workflow.steps import TrainingStep, ProcessingStep

# Define pipeline steps
processing_step = ProcessingStep(
    name='data-processing',
    processor=processor,
    inputs=[...],
    outputs=[...]
)

training_step = TrainingStep(
    name='model-training',
    estimator=estimator,
    inputs=[...]
)

# Create pipeline
pipeline = Pipeline(
    name='house-price-pipeline',
    steps=[processing_step, training_step]
)

pipeline.upsert()
pipeline.start()

from sagemaker.model_monitor import DataCaptureConfig

# Deploy model with data capture
predictor = model.deploy(
    initial_instance_count=1,
    instance_type='ml.m5.large',
    data_capture_config=DataCaptureConfig(
        enable_capture=True,
        sampling_percentage=100,
        destination_s3_uri=s3_capture_path
    )
)

# Set up monitoring
from sagemaker.model_monitor import ModelMonitor

monitor = ModelMonitor(
    role=sagemaker.get_execution_role(),
    image_uri=sagemaker.image_uris.retrieve('model-monitor', region),
    instance_count=1,
    instance_type='ml.m5.large'
)

monitor.create_monitoring_schedule(
    monitor_schedule_name='house-price-monitor',
    endpoint_input=predictor.endpoint_name,
    schedule_cron_expression='cron(0 * ? * * *)'  # Hourly
)