diff --git a/docs/sections/api/apidocs/io-managers.rst b/docs/sections/api/apidocs/io-managers.rst --- a/docs/sections/api/apidocs/io-managers.rst +++ b/docs/sections/api/apidocs/io-managers.rst @@ -6,13 +6,13 @@ IO Managers --------------- -IO managers are user-provided objects that specify how to store step outputs and load step inputs. +IO managers are user-provided objects that store solid outputs and load them as inputs to downstream +solids. .. autodecorator:: io_manager .. autoclass:: IOManager :members: - :show-inheritance: .. autoclass:: IOManagerDefinition :members: @@ -31,7 +31,8 @@ Root Input Managers -------------- -Root input managers are user-provided objects that specify how to load step inputs. +Root input managers are user-provided objects that specify how to load inputs that aren't connected +to upstream outputs. .. autodecorator:: root_input_manager diff --git a/python_modules/dagster/dagster/core/definitions/input.py b/python_modules/dagster/dagster/core/definitions/input.py --- a/python_modules/dagster/dagster/core/definitions/input.py +++ b/python_modules/dagster/dagster/core/definitions/input.py @@ -57,8 +57,8 @@ :py:func:`PythonObjectDagsterType`, or a Python type. Defaults to :py:class:`Any`. description (Optional[str]): Human-readable description of the input. default_value (Optional[Any]): The default value to use if no input is provided. - root_manager_key (Optional[str]): The resource key for the RootInputManager used for loading - this input when it is not hooked up to an upstream output. + root_manager_key (Optional[str]): The resource key for the :py:class:`RootInputManager` used + for loading this input when it is not connected to an upstream output. metadata (Optional[Dict[str, Any]]): A dict of metadata for the input. """ diff --git a/python_modules/dagster/dagster/core/execution/context/system.py b/python_modules/dagster/dagster/core/execution/context/system.py --- a/python_modules/dagster/dagster/core/execution/context/system.py +++ b/python_modules/dagster/dagster/core/execution/context/system.py @@ -444,6 +444,8 @@ ) ): """ + The context object that is available to the `handle_output` method of an :py:class:`IOManager`. + Attributes: step_key (str): The step_key for the compute step that produced the output. name (str): The name of the output that produced the output. @@ -455,7 +457,7 @@ config (Optional[Any]): The configuration for the output. solid_def (Optional[SolidDefinition]): The definition of the solid that produced the output. dagster_type (Optional[DagsterType]): The type of this output. - log (Optional[DagsterLogmanager]): The log manager to use for this output. + log (Optional[DagsterLogManager]): The log manager to use for this output. version (Optional[str]): (Experimental) The version of the output. resources (Optional[ScopedResources]): The resources required by the output manager, specified by the `required_resource_keys` parameter. @@ -531,7 +533,7 @@ ) ): """ - The ``context`` object available to the load method of :py:class:`RootInputManager`. + The ``context`` object available to the load_input method of :py:class:`RootInputManager`. Attributes: name (Optional[str]): The name of the input that we're loading. diff --git a/python_modules/dagster/dagster/core/storage/input_manager.py b/python_modules/dagster/dagster/core/storage/input_manager.py new file mode 100644 --- /dev/null +++ b/python_modules/dagster/dagster/core/storage/input_manager.py @@ -0,0 +1,18 @@ +from abc import ABC, abstractmethod + + +class InputManager(ABC): + """ + Base interface for classes that are responsible for loading solid inputs. + """ + + @abstractmethod + def load_input(self, context): + """The user-defined read method that loads an input to a solid. + + Args: + context (InputContext): The input context. + + Returns: + Any: The data object. + """ diff --git a/python_modules/dagster/dagster/core/storage/io_manager.py b/python_modules/dagster/dagster/core/storage/io_manager.py --- a/python_modules/dagster/dagster/core/storage/io_manager.py +++ b/python_modules/dagster/dagster/core/storage/io_manager.py @@ -1,3 +1,4 @@ +from abc import abstractmethod from functools import update_wrapper from dagster import check @@ -6,15 +7,21 @@ convert_user_facing_definition_config_schema, ) from dagster.core.definitions.resource import ResourceDefinition +from dagster.core.storage.input_manager import InputManager from dagster.core.storage.output_manager import IOutputManagerDefinition, OutputManager from dagster.core.storage.root_input_manager import IInputManagerDefinition class IOManagerDefinition(ResourceDefinition, IInputManagerDefinition, IOutputManagerDefinition): - """Definition of an output manager resource. + """Definition of an IO manager resource. - An OutputManagerDefinition is a :py:class:`ResourceDefinition` whose resource_fn returns an + IOManagers are used to store solid outputs and load them as inputs to downstream solids. + + An IOManagerDefinition is a :py:class:`ResourceDefinition` whose `resource_fn` returns an :py:class:`IOManager`. + + The easiest way to create an IOManagerDefnition is with the :py:func:`@io_manager ` + decorator. """ def __init__( @@ -57,14 +64,37 @@ ) -class IOManager(OutputManager): +class IOManager(InputManager, OutputManager): """ Base class for user-provided IO managers. + IOManagers are used to store solid outputs and load them as inputs to downstream solids. + Extend this class to handle how objects are loaded and stored. Users should implement ``handle_output`` to store an object and ``load_input`` to retrieve an object. """ + @abstractmethod + def load_input(self, context): + """User-defined method that loads an input to a solid. + + Args: + context (InputContext): The input context, which describes the input that's being loaded + and the upstream output that's being loaded from. + + Returns: + Any: The data object. + """ + + @abstractmethod + def handle_output(self, context, obj): + """User-defined method that stores an output of a solid. + + Args: + context (OutputContext): The context of the step output that produces this object. + obj (Any): The object, returned by the solid, to be stored. + """ + def io_manager( config_schema=None, @@ -77,8 +107,10 @@ """ Define an IO manager. - The decorated function should accept an :py:class:`InitResourceContext and return an - py:class:`IOManager`. + IOManagers are used to store solid outputs and load them as inputs to downstream solids. + + The decorated function should accept an :py:class:`InitResourceContext` and return an + :py:class:`IOManager`. Args: config_schema (Optional[ConfigSchema]): The schema for the resource config. Configuration diff --git a/python_modules/dagster/dagster/core/storage/output_manager.py b/python_modules/dagster/dagster/core/storage/output_manager.py --- a/python_modules/dagster/dagster/core/storage/output_manager.py +++ b/python_modules/dagster/dagster/core/storage/output_manager.py @@ -13,7 +13,7 @@ @abstractproperty def output_config_schema(self): """The schema for per-output configuration for outputs that are managed by this - input manager""" + manager""" class OutputManagerDefinition(ResourceDefinition, IOutputManagerDefinition): @@ -67,7 +67,7 @@ @abstractmethod def handle_output(self, context, obj): - """Handles an output produced by a solid. Often, this means materializing it to persistent + """Handles an output produced by a solid. Usually, this means materializing it to persistent storage. Args: diff --git a/python_modules/dagster/dagster/core/storage/root_input_manager.py b/python_modules/dagster/dagster/core/storage/root_input_manager.py --- a/python_modules/dagster/dagster/core/storage/root_input_manager.py +++ b/python_modules/dagster/dagster/core/storage/root_input_manager.py @@ -1,4 +1,4 @@ -from abc import ABC, abstractmethod, abstractproperty +from abc import abstractmethod, abstractproperty from functools import update_wrapper from dagster import check @@ -7,6 +7,7 @@ convert_user_facing_definition_config_schema, ) from dagster.core.definitions.resource import ResourceDefinition +from dagster.core.storage.input_manager import InputManager class IInputManagerDefinition: @@ -19,8 +20,13 @@ class RootInputManagerDefinition(ResourceDefinition, IInputManagerDefinition): """Definition of a root input manager resource. + Root input managers load solid inputs that aren't connected to upstream outputs. + An RootInputManagerDefinition is a :py:class:`ResourceDefinition` whose resource_fn returns an - :py:class:`RootInputManager`. RootInputManagers are used to load the inputs to solids. + :py:class:`RootInputManager`. + + The easiest way to create an RootInputManagerDefinition is with the + :py:func:`@root_input_manager ` decorator. """ def __init__( @@ -58,10 +64,11 @@ ) -class RootInputManager(ABC): - """RootInputManagers are used to load the inputs to solids at the root of a pipeline. +class RootInputManager(InputManager): + """RootInputManagers are used to load inputs to solids at the root of a pipeline. - The easiest way to define an RootInputManager is with the :py:function:`root_input_manager` decorator. + The easiest way to define an RootInputManager is with the + :py:func:`@root_input_manager ` decorator. """ @abstractmethod @@ -85,6 +92,8 @@ ): """Define a root input manager. + Root input managers load solid inputs that aren't connected to upstream outputs. + The decorated function should accept a :py:class:`InputContext` and resource config, and return a loaded object that will be passed into one of the inputs of a solid.