Apache Mesos

# Exemplo de Framework Scheduler em Python
import mesos.interface
from mesos.interface import mesos_pb2
import mesos.native

class ExampleScheduler(mesos.interface.Scheduler):
    def __init__(self):
        self.tasksLaunched = 0
        self.tasksFinished = 0
        self.totalTasks = 5

    def registered(self, driver, frameworkId, masterInfo):
        print("Registered with framework ID %s" % frameworkId.value)

    def resourceOffers(self, driver, offers):
        for offer in offers:
            tasks = []
            
            # Verifica recursos disponíveis
            cpus = self.getResource(offer.resources, "cpus")
            mem = self.getResource(offer.resources, "mem")
            
            if cpus >= 1 and mem >= 128 and self.tasksLaunched < self.totalTasks:
                # Cria uma nova tarefa
                task = mesos_pb2.TaskInfo()
                task.task_id.value = str(self.tasksLaunched)
                task.slave_id.value = offer.slave_id.value
                task.name = "task %d" % self.tasksLaunched
                
                # Define comando a ser executado
                task.command.value = "echo 'Hello from Mesos task %d'" % self.tasksLaunched
                
                # Define recursos necessários
                cpuResource = task.resources.add()
                cpuResource.name = "cpus"
                cpuResource.type = mesos_pb2.Value.SCALAR
                cpuResource.scalar.value = 1
                
                memResource = task.resources.add()
                memResource.name = "mem"
                memResource.type = mesos_pb2.Value.SCALAR
                memResource.scalar.value = 128
                
                tasks.append(task)
                self.tasksLaunched += 1
            
            # Lança tarefas
            driver.launchTasks(offer.id, tasks)

    def statusUpdate(self, driver, update):
        print("Task %s is in state %s" % (update.task_id.value, 
                                         mesos_pb2.TaskState.Name(update.state)))
        
        if update.state == mesos_pb2.TASK_FINISHED:
            self.tasksFinished += 1
            
        if self.tasksFinished == self.totalTasks:
            print("All tasks completed, stopping framework")
            driver.stop()

    def getResource(self, resources, name):
        for resource in resources:
            if resource.name == name:
                return resource.scalar.value
        return 0

if __name__ == "__main__":
    # Configuração do framework
    framework = mesos_pb2.FrameworkInfo()
    framework.user = ""  # Usuário atual
    framework.name = "Example Framework"
    
    # Cria driver
    driver = mesos.native.MesosSchedulerDriver(
        ExampleScheduler(),
        framework,
        "zk://localhost:2181/mesos"  # Master endpoint
    )
    
    # Executa framework
    status = driver.run()
    print("Framework finished with status:", status)

{
  "id": "/web-app",
  "cmd": "python app.py",
  "cpus": 1.0,
  "mem": 512,
  "instances": 3,
  "container": {
    "type": "DOCKER",
    "docker": {
      "image": "myapp:latest",
      "network": "BRIDGE",
      "portMappings": [
        {
          "containerPort": 8080,
          "hostPort": 0,
          "protocol": "tcp"
        }
      ]
    }
  },
  "healthChecks": [
    {
      "protocol": "HTTP",
      "path": "/health",
      "portIndex": 0,
      "timeoutSeconds": 10,
      "gracePeriodSeconds": 10,
      "intervalSeconds": 2,
      "maxConsecutiveFailures": 10
    }
  ],
  "upgradeStrategy": {
    "minimumHealthCapacity": 0.5,
    "maximumOverCapacity": 0.2
  }
}

# Configuração do Mesos Master
# /etc/mesos-master/quorum
2

# /etc/mesos-master/ip
192.168.1.10

# /etc/mesos-master/hostname
mesos-master-1

# /etc/mesos-master/cluster
production-cluster

# Configuração do ZooKeeper
# /etc/mesos/zk
zk://192.168.1.10:2181,192.168.1.11:2181,192.168.1.12:2181/mesos

# Iniciar Mesos Master
sudo service mesos-master start

# Configuração do Mesos Slave
# /etc/mesos-slave/ip
192.168.1.20

# /etc/mesos-slave/hostname
mesos-slave-1

# /etc/mesos-slave/resources
cpus:4;mem:8192;disk:100000;ports:[8000-9000]

# /etc/mesos-slave/attributes
rack:rack1;datacenter:dc1

# Iniciar Mesos Slave
sudo service mesos-slave start

# Configuração do Spark para usar Mesos
export MESOS_NATIVE_JAVA_LIBRARY=/usr/local/lib/libmesos.so

# Submeter job Spark no Mesos
spark-submit \
  --master mesos://zk://localhost:2181/mesos \
  --deploy-mode cluster \
  --class org.apache.spark.examples.SparkPi \
  --conf spark.mesos.executor.docker.image=mesosphere/spark:2.4.0 \
  --conf spark.mesos.executor.home=/opt/spark \
  --conf spark.mesos.coarse=true \
  --conf spark.cores.max=4 \
  --conf spark.executor.memory=2g \
  examples/jars/spark-examples_2.12-2.4.0.jar 100

# Script para monitorar métricas do Mesos
import requests
import json

def get_mesos_metrics():
    """Coleta métricas do Mesos Master"""
    try:
        response = requests.get('http://mesos-master:5050/metrics/snapshot')
        metrics = response.json()
        
        # Métricas importantes
        important_metrics = {
            'master/slaves_active': metrics.get('master/slaves_active', 0),
            'master/slaves_inactive': metrics.get('master/slaves_inactive', 0),
            'master/frameworks_active': metrics.get('master/frameworks_active', 0),
            'master/tasks_running': metrics.get('master/tasks_running', 0),
            'master/tasks_failed': metrics.get('master/tasks_failed', 0),
            'master/cpus_total': metrics.get('master/cpus_total', 0),
            'master/cpus_used': metrics.get('master/cpus_used', 0),
            'master/mem_total': metrics.get('master/mem_total', 0),
            'master/mem_used': metrics.get('master/mem_used', 0)
        }
        
        return important_metrics
        
    except Exception as e:
        print(f"Erro ao coletar métricas: {e}")
        return None

def print_cluster_status():
    """Imprime status do cluster"""
    metrics = get_mesos_metrics()
    if metrics:
        print("=== Status do Cluster Mesos ===")
        print(f"Slaves Ativos: {metrics['master/slaves_active']}")
        print(f"Slaves Inativos: {metrics['master/slaves_inactive']}")
        print(f"Frameworks Ativos: {metrics['master/frameworks_active']}")
        print(f"Tarefas Executando: {metrics['master/tasks_running']}")
        print(f"Tarefas Falharam: {metrics['master/tasks_failed']}")
        print(f"CPU Utilização: {metrics['master/cpus_used']}/{metrics['master/cpus_total']}")
        print(f"Memória Utilização: {metrics['master/mem_used']:.0f}MB/{metrics['master/mem_total']:.0f}MB")

if __name__ == "__main__":
    print_cluster_status()