diff --git a/python_modules/dagster/dagster/core/definitions/handle.py b/python_modules/dagster/dagster/core/definitions/handle.py --- a/python_modules/dagster/dagster/core/definitions/handle.py +++ b/python_modules/dagster/dagster/core/definitions/handle.py @@ -7,7 +7,7 @@ from collections import namedtuple from enum import Enum -from dagster import check +from dagster import check, seven from dagster.core.definitions.pipeline import PipelineDefinition from dagster.core.definitions.repository import RepositoryDefinition from dagster.core.errors import DagsterInvariantViolationError @@ -148,11 +148,43 @@ @staticmethod def from_file_target(python_file, fn_name, from_handle=None): file_directory = os.path.dirname(python_file) - if file_directory not in sys.path: + + is_module = os.path.isfile(os.path.join(file_directory, '__init__.py')) + # If the file target is in fact in a module, we really don't want to add its directory to + # sys.path. This can lead to all kinds of subtle, nasty, shadowing errors with very + # obscure error messages. See: + # http://python-notes.curiousefficiency.org/en/latest/python_concepts/import_traps.html#the-double-import-trap + if file_directory not in sys.path: # and not is_module: sys.path.append(file_directory) - module_name = os.path.splitext(os.path.basename(python_file))[0] - module = imp.load_source(module_name, python_file) + try: + module_name = os.path.splitext(os.path.basename(python_file))[0] + + # This is a questionable bit of magic, but even members of the core dev team have been + # observed apotropaically renaming files and moving directories around in response to + # the status quo ante behavior, where imp.load_source would fail with an + # `ImportError: attempted relative import with no known parent package` on + # apparently valid relative imports in the target file, which appeared to be part of a + # valid package. + try: + module = imp.load_source(module_name, python_file) + except ImportError as exc: + if not is_module: + raise exc + + path, name = os.path.split(os.path.abspath(python_file)) + module = [name] + while os.path.isfile(os.path.join(path, '__init__.py')): + path, name = os.path.split(path) + module.append(name) + module_name = '.'.join(reversed(module)) + try: + module = importlib.import_module(module_name) + except seven.ModuleNotFoundError: + raise exc + finally: + if file_directory in sys.path: + sys.path.remove(file_directory) return LoaderEntrypoint(module, module_name, fn_name, from_handle)