diff --git a/docs/next/src/pages/overview/io-managers/io-managers.mdx b/docs/next/src/pages/overview/io-managers/io-managers.mdx
--- a/docs/next/src/pages/overview/io-managers/io-managers.mdx
+++ b/docs/next/src/pages/overview/io-managers/io-managers.mdx
@@ -28,9 +28,11 @@
IOManagers are [resources](/overview/modes-resources-presets/modes-resources), which means users can supply different IOManagers for the same solid outputs in different situations. For example, you might use an in-memory IOManager for unit-testing a pipeline and an S3IOManager in production.
-## Input managers
+## Root input managers (experimental)
-s are user-provided objects that are responsible for loading the inputs of solids whose inputs are not hooked up to an upstream solid's output.
+s are user-provided objects that are responsible for loading the inputs of solids whose inputs are not connected to an upstream solid's output.
+
+Root input managers are currently an experimental feature.
## Setting a pipeline-wide IO manager
@@ -113,7 +115,7 @@
endBefore:execute_end_marker
```
-## Providing an input manager for a root input
+## Providing an input manager for a root input (experimental)
When you have a solid at the beginning of a pipeline that operates on data from external source, you might wish to separate that I/O from your solid's business logic, in the same way you would with an IO manager if the solid were loading from an upstream output.
@@ -130,7 +132,7 @@
The decorator behaves nearly identically to the decorator. It yields an , which is a that will produce an .
-## Providing per-input config to a root input manager
+## Providing per-input config to a root input manager (experimental)
When launching a run, you might want to parameterize how particular root inputs are loaded.
@@ -150,7 +152,7 @@
endBefore:execute_end_marker
```
-## Using a root input manager with subselection
+## Using a root input manager with subselection (experimental)
You might want to execute a subset of solids in your pipeline and control how the inputs of those solids are loaded. Root input managers also help in these situations, because the inputs at the beginning of the subset become the new "roots".
diff --git a/docs/sections/api/apidocs/io-managers.rst b/docs/sections/api/apidocs/io-managers.rst
--- a/docs/sections/api/apidocs/io-managers.rst
+++ b/docs/sections/api/apidocs/io-managers.rst
@@ -6,13 +6,13 @@
IO Managers
---------------
-IO managers are user-provided objects that specify how to store step outputs and load step inputs.
+IO managers are user-provided objects that store solid outputs and load them as inputs to downstream
+solids.
.. autodecorator:: io_manager
.. autoclass:: IOManager
:members:
- :show-inheritance:
.. autoclass:: IOManagerDefinition
:members:
@@ -28,19 +28,6 @@
:members:
-Root Input Managers
---------------
-
-Root input managers are user-provided objects that specify how to load step inputs.
-
-.. autodecorator:: root_input_manager
-
-.. autoclass:: RootInputManager
- :members:
-
-.. autoclass:: RootInputManagerDefinition
- :members:
-
Built-in IO Managers
------------------------
@@ -50,3 +37,18 @@
.. autodata:: fs_io_manager
.. autodata:: custom_path_fs_io_manager
+
+
+Root Input Managers (Experimental)
+--------------
+
+Root input managers are user-provided objects that specify how to load inputs that aren't connected
+to upstream outputs.
+
+.. autodecorator:: root_input_manager
+
+.. autoclass:: RootInputManager
+ :members:
+
+.. autoclass:: RootInputManagerDefinition
+ :members:
\ No newline at end of file
diff --git a/python_modules/dagster/dagster/core/definitions/input.py b/python_modules/dagster/dagster/core/definitions/input.py
--- a/python_modules/dagster/dagster/core/definitions/input.py
+++ b/python_modules/dagster/dagster/core/definitions/input.py
@@ -7,6 +7,7 @@
DagsterType,
resolve_dagster_type,
)
+from dagster.utils.backcompat import experimental_arg_warning
from .utils import check_valid_name
@@ -57,8 +58,9 @@
:py:func:`PythonObjectDagsterType`, or a Python type. Defaults to :py:class:`Any`.
description (Optional[str]): Human-readable description of the input.
default_value (Optional[Any]): The default value to use if no input is provided.
- root_manager_key (Optional[str]): The resource key for the RootInputManager used for loading
- this input when it is not hooked up to an upstream output.
+ root_manager_key (Optional[str]): (Experimental) The resource key for the
+ :py:class:`RootInputManager` used for loading this input when it is not connected to an
+ upstream output.
metadata (Optional[Dict[str, Any]]): A dict of metadata for the input.
"""
@@ -80,6 +82,9 @@
self._default_value = _check_default_value(self._name, self._dagster_type, default_value)
+ if root_manager_key:
+ experimental_arg_warning("root_manager_key", "InputDefinition")
+
self._root_manager_key = check.opt_str_param(root_manager_key, "root_manager_key")
self._metadata = check.opt_dict_param(metadata, "metadata", key_type=str)
diff --git a/python_modules/dagster/dagster/core/execution/context/system.py b/python_modules/dagster/dagster/core/execution/context/system.py
--- a/python_modules/dagster/dagster/core/execution/context/system.py
+++ b/python_modules/dagster/dagster/core/execution/context/system.py
@@ -444,6 +444,8 @@
)
):
"""
+ The context object that is available to the `handle_output` method of an :py:class:`IOManager`.
+
Attributes:
step_key (str): The step_key for the compute step that produced the output.
name (str): The name of the output that produced the output.
@@ -455,7 +457,7 @@
config (Optional[Any]): The configuration for the output.
solid_def (Optional[SolidDefinition]): The definition of the solid that produced the output.
dagster_type (Optional[DagsterType]): The type of this output.
- log (Optional[DagsterLogmanager]): The log manager to use for this output.
+ log (Optional[DagsterLogManager]): The log manager to use for this output.
version (Optional[str]): (Experimental) The version of the output.
resources (Optional[ScopedResources]): The resources required by the output manager, specified by the
`required_resource_keys` parameter.
@@ -531,7 +533,7 @@
)
):
"""
- The ``context`` object available to the load method of :py:class:`RootInputManager`.
+ The ``context`` object available to the load_input method of :py:class:`RootInputManager`.
Attributes:
name (Optional[str]): The name of the input that we're loading.
diff --git a/python_modules/dagster/dagster/core/storage/input_manager.py b/python_modules/dagster/dagster/core/storage/input_manager.py
new file mode 100644
--- /dev/null
+++ b/python_modules/dagster/dagster/core/storage/input_manager.py
@@ -0,0 +1,18 @@
+from abc import ABC, abstractmethod
+
+
+class InputManager(ABC):
+ """
+ Base interface for classes that are responsible for loading solid inputs.
+ """
+
+ @abstractmethod
+ def load_input(self, context):
+ """The user-defined read method that loads an input to a solid.
+
+ Args:
+ context (InputContext): The input context.
+
+ Returns:
+ Any: The data object.
+ """
diff --git a/python_modules/dagster/dagster/core/storage/io_manager.py b/python_modules/dagster/dagster/core/storage/io_manager.py
--- a/python_modules/dagster/dagster/core/storage/io_manager.py
+++ b/python_modules/dagster/dagster/core/storage/io_manager.py
@@ -1,3 +1,4 @@
+from abc import abstractmethod
from functools import update_wrapper
from dagster import check
@@ -6,15 +7,21 @@
convert_user_facing_definition_config_schema,
)
from dagster.core.definitions.resource import ResourceDefinition
+from dagster.core.storage.input_manager import InputManager
from dagster.core.storage.output_manager import IOutputManagerDefinition, OutputManager
from dagster.core.storage.root_input_manager import IInputManagerDefinition
class IOManagerDefinition(ResourceDefinition, IInputManagerDefinition, IOutputManagerDefinition):
- """Definition of an output manager resource.
+ """Definition of an IO manager resource.
- An OutputManagerDefinition is a :py:class:`ResourceDefinition` whose resource_fn returns an
+ IOManagers are used to store solid outputs and load them as inputs to downstream solids.
+
+ An IOManagerDefinition is a :py:class:`ResourceDefinition` whose `resource_fn` returns an
:py:class:`IOManager`.
+
+ The easiest way to create an IOManagerDefnition is with the :py:func:`@io_manager `
+ decorator.
"""
def __init__(
@@ -57,14 +64,37 @@
)
-class IOManager(OutputManager):
+class IOManager(InputManager, OutputManager):
"""
Base class for user-provided IO managers.
+ IOManagers are used to store solid outputs and load them as inputs to downstream solids.
+
Extend this class to handle how objects are loaded and stored. Users should implement
``handle_output`` to store an object and ``load_input`` to retrieve an object.
"""
+ @abstractmethod
+ def load_input(self, context):
+ """User-defined method that loads an input to a solid.
+
+ Args:
+ context (InputContext): The input context, which describes the input that's being loaded
+ and the upstream output that's being loaded from.
+
+ Returns:
+ Any: The data object.
+ """
+
+ @abstractmethod
+ def handle_output(self, context, obj):
+ """User-defined method that stores an output of a solid.
+
+ Args:
+ context (OutputContext): The context of the step output that produces this object.
+ obj (Any): The object, returned by the solid, to be stored.
+ """
+
def io_manager(
config_schema=None,
@@ -77,8 +107,10 @@
"""
Define an IO manager.
- The decorated function should accept an :py:class:`InitResourceContext and return an
- py:class:`IOManager`.
+ IOManagers are used to store solid outputs and load them as inputs to downstream solids.
+
+ The decorated function should accept an :py:class:`InitResourceContext` and return an
+ :py:class:`IOManager`.
Args:
config_schema (Optional[ConfigSchema]): The schema for the resource config. Configuration
diff --git a/python_modules/dagster/dagster/core/storage/output_manager.py b/python_modules/dagster/dagster/core/storage/output_manager.py
--- a/python_modules/dagster/dagster/core/storage/output_manager.py
+++ b/python_modules/dagster/dagster/core/storage/output_manager.py
@@ -13,7 +13,7 @@
@abstractproperty
def output_config_schema(self):
"""The schema for per-output configuration for outputs that are managed by this
- input manager"""
+ manager"""
class OutputManagerDefinition(ResourceDefinition, IOutputManagerDefinition):
@@ -67,7 +67,7 @@
@abstractmethod
def handle_output(self, context, obj):
- """Handles an output produced by a solid. Often, this means materializing it to persistent
+ """Handles an output produced by a solid. Usually, this means materializing it to persistent
storage.
Args:
diff --git a/python_modules/dagster/dagster/core/storage/root_input_manager.py b/python_modules/dagster/dagster/core/storage/root_input_manager.py
--- a/python_modules/dagster/dagster/core/storage/root_input_manager.py
+++ b/python_modules/dagster/dagster/core/storage/root_input_manager.py
@@ -1,4 +1,4 @@
-from abc import ABC, abstractmethod, abstractproperty
+from abc import abstractmethod, abstractproperty
from functools import update_wrapper
from dagster import check
@@ -7,6 +7,8 @@
convert_user_facing_definition_config_schema,
)
from dagster.core.definitions.resource import ResourceDefinition
+from dagster.core.storage.input_manager import InputManager
+from dagster.utils.backcompat import experimental
class IInputManagerDefinition:
@@ -19,8 +21,13 @@
class RootInputManagerDefinition(ResourceDefinition, IInputManagerDefinition):
"""Definition of a root input manager resource.
+ Root input managers load solid inputs that aren't connected to upstream outputs.
+
An RootInputManagerDefinition is a :py:class:`ResourceDefinition` whose resource_fn returns an
- :py:class:`RootInputManager`. RootInputManagers are used to load the inputs to solids.
+ :py:class:`RootInputManager`.
+
+ The easiest way to create an RootInputManagerDefinition is with the
+ :py:func:`@root_input_manager ` decorator.
"""
def __init__(
@@ -58,10 +65,11 @@
)
-class RootInputManager(ABC):
- """RootInputManagers are used to load the inputs to solids at the root of a pipeline.
+class RootInputManager(InputManager):
+ """RootInputManagers are used to load inputs to solids at the root of a pipeline.
- The easiest way to define an RootInputManager is with the :py:function:`root_input_manager` decorator.
+ The easiest way to define an RootInputManager is with the
+ :py:func:`@root_input_manager ` decorator.
"""
@abstractmethod
@@ -76,6 +84,7 @@
"""
+@experimental
def root_input_manager(
config_schema=None,
description=None,
@@ -85,6 +94,8 @@
):
"""Define a root input manager.
+ Root input managers load solid inputs that aren't connected to upstream outputs.
+
The decorated function should accept a :py:class:`InputContext` and resource config, and return
a loaded object that will be passed into one of the inputs of a solid.