Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

BaseExecutor allows executors to consumer task_instance parameters #44016

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
+24 −1
Draft

BaseExecutor allows executors to consumer task_instance parameters #44016

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1134e2b
First idea to add new run function
Nov 12, 2024
4b8ef13
Renamed function to execute
Nov 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 24 additions & 1 deletion airflow/executors/base_executor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import logging
import sys
import warnings
from collections import defaultdict, deque
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Tuple
Expand Down Expand Up @@ -399,7 +400,7 @@ def _process_tasks(self, task_tuples: list[TaskTuple]) -> None:
span.set_attribute("queue", str(queue))
span.set_attribute("executor_config", str(executor_config))
del self.queued_tasks[key]
self.execute_async(key=key, command=command, queue=queue, executor_config=executor_config)
self.execute(ti=task_instance, command=command, queue=queue, executor_config=executor_config)
self.running.add(key)

def change_state(
Expand Down Expand Up @@ -505,6 +506,28 @@ def get_event_buffer(self, dag_ids=None) -> dict[TaskInstanceKey, EventBufferVal

return cleared_events

def execute(
self,
ti: TaskInstance,
command: CommandType,
queue: str | None = None,
executor_config: Any | None = None,
) -> None: # pragma: no cover
"""
Execute the command.

:param ti: task instance to run
:param command: Command to run
:param queue: name of the queue
:param executor_config: Configuration passed to the executor.
"""
warnings.warn(
f"Old execute_async function is used. Please update your executor: {type(self).__name__} to use new execute function with updated interface.",
DeprecationWarning,
stacklevel=2,
)
self.execute_async(key=ti.key, command=command, queue=queue, executor_config=executor_config)

def execute_async(
self,
key: TaskInstanceKey,
Expand Down