Apache Oozie

Visão Geral

Apache Oozie é um sistema de orquestração de workflows para gerenciar jobs do Apache Hadoop. Permite coordenar múltiplos jobs MapReduce, Pig, Hive e outros componentes do ecossistema Hadoop em workflows complexos.

Desenvolvido pelo Yahoo! e posteriormente doado à Apache Software Foundation, Oozie é especialmente útil para pipelines de dados que requerem execução sequencial ou paralela de múltiplas tarefas Hadoop.

Principais Características

Exemplo de Workflow

Exemplo de um workflow Oozie simples que executa um job MapReduce:

<workflow-app xmlns="uri:oozie:workflow:0.5" name="sample-workflow">
    <start to="mapreduce-job"/>
    
    <action name="mapreduce-job">
        <map-reduce>
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <configuration>
                <property>
                    <name>mapred.mapper.class</name>
                    <value>org.apache.hadoop.examples.WordCount$TokenizerMapper</value>
                </property>
                <property>
                    <name>mapred.reducer.class</name>
                    <value>org.apache.hadoop.examples.WordCount$IntSumReducer</value>
                </property>
                <property>
                    <name>mapred.input.dir</name>
                    <value>${inputDir}</value>
                </property>
                <property>
                    <name>mapred.output.dir</name>
                    <value>${outputDir}</value>
                </property>
            </configuration>
        </map-reduce>
        <ok to="end"/>
        <error to="fail"/>
    </action>
    
    <kill name="fail">
        <message>Workflow failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>
    
    <end name="end"/>
</workflow-app>

nameNode=hdfs://localhost:9000
jobTracker=localhost:9001
queueName=default
examplesRoot=examples

oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/map-reduce
inputDir=${nameNode}/user/${user.name}/${examplesRoot}/input-data/text
outputDir=${nameNode}/user/${user.name}/${examplesRoot}/output-data/${outputDir}

Exemplo de Coordinator

Oozie Coordinator permite agendar workflows para execução periódica:

<coordinator-app name="daily-workflow" 
                 frequency="${coord:days(1)}"
                 start="${start}" end="${end}" 
                 timezone="UTC"
                 xmlns="uri:oozie:coordinator:0.4">
    
    <datasets>
        <dataset name="input-data" frequency="${coord:days(1)}"
                 initial-instance="2024-01-01T00:00Z" timezone="UTC">
            <uri-template>${nameNode}/data/input/${YEAR}/${MONTH}/${DAY}</uri-template>
        </dataset>
        
        <dataset name="output-data" frequency="${coord:days(1)}"
                 initial-instance="2024-01-01T00:00Z" timezone="UTC">
            <uri-template>${nameNode}/data/output/${YEAR}/${MONTH}/${DAY}</uri-template>
        </dataset>
    </datasets>
    
    <input-events>
        <data-in name="input" dataset="input-data">
            <instance>${coord:current(0)}</instance>
        </data-in>
    </input-events>
    
    <output-events>
        <data-out name="output" dataset="output-data">
            <instance>${coord:current(0)}</instance>
        </data-out>
    </output-events>
    
    <action>
        <workflow>
            <app-path>${workflowAppUri}</app-path>
            <configuration>
                <property>
                    <name>inputDir</name>
                    <value>${coord:dataIn('input')}</value>
                </property>
                <property>
                    <name>outputDir</name>
                    <value>${coord:dataOut('output')}</value>
                </property>
            </configuration>
        </workflow>
    </action>
</coordinator-app>

Casos de Uso

Alternativas Modernas