diff --git a/CHANGES.md b/CHANGES.md --- a/CHANGES.md +++ b/CHANGES.md @@ -1,26 +1,31 @@ # Changelog +## 0.8.7 (Upcoming) + +**New** + +- Dagit can be hosted on a sub-path by passing `--path-prefix` to the dagit CLI. #2073 + ## 0.8.6 **Breaking Changes** - The `dagster-celery` module has been broken apart to manage dependencies more coherently. There -are now three modules: `dagster-celery`, `dagster-celery-k8s`, and `dagster-celery-docker`. + are now three modules: `dagster-celery`, `dagster-celery-k8s`, and `dagster-celery-docker`. - Related to above, the `dagster-celery worker start` command now takes a required `-A` parameter -which must point to the `app.py` file within the appropriate module. E.g if you are using the -`celery_k8s_job_executor` then you must use the `-A dagster_celery_k8s.app` option when using the -`celery` or `dagster-celery` cli tools. Similar for the `celery_docker_executor`: -`-A dagster_celery_docker.app` must be used. + which must point to the `app.py` file within the appropriate module. E.g if you are using the + `celery_k8s_job_executor` then you must use the `-A dagster_celery_k8s.app` option when using the + `celery` or `dagster-celery` cli tools. Similar for the `celery_docker_executor`: + `-A dagster_celery_docker.app` must be used. - Renamed the `input_hydration_config` and `output_materialization_config` decorators to -`dagster_type_` and `dagster_type_materializer` respectively. Renamed DagsterType's -`input_hydration_config` and `output_materialization_config` arguments to `loader` and `materializer` respectively. + `dagster_type_` and `dagster_type_materializer` respectively. Renamed DagsterType's + `input_hydration_config` and `output_materialization_config` arguments to `loader` and `materializer` respectively. **New** - New pipeline scoped runs tab in Dagit - Add the following Dask Job Queue clusters: moab, sge, lsf, slurm, oar (thanks @DavidKatz-il!) -- K8s resource-requirements for run coordinator pods can be specified using the `dagster-k8s/ -resource_requirements` tag on pipeline definitions: +- K8s resource-requirements for run coordinator pods can be specified using the `dagster-k8s/ resource_requirements` tag on pipeline definitions: ```python @pipeline( @@ -33,12 +38,12 @@ ) def foo_bar_pipeline(): ``` + - Added better error messaging in dagit for partition set and schedule configuration errors - An initial version of the CeleryDockerExecutor was added (thanks @mrdrprofuroboros!). The celery -workers will launch tasks in docker containers. -- **Experimental:** Great Expectations integration is currently under development in the new library -dagster-ge. Example usage can be found [here]( -https://github.com/dagster-io/dagster/blob/master/python_modules/libraries/dagster-ge/dagster_ge/examples/ge_demo.py) + workers will launch tasks in docker containers. +- **Experimental:** Great Expectations integration is currently under development in the new library + dagster-ge. Example usage can be found [here](https://github.com/dagster-io/dagster/blob/master/python_modules/libraries/dagster-ge/dagster_ge/examples/ge_demo.py) ## 0.8.5 diff --git a/examples/docs_snippets/docs_snippets_tests/intro_tutorial_tests/test_cli_invocations.py b/examples/docs_snippets/docs_snippets_tests/intro_tutorial_tests/test_cli_invocations.py --- a/examples/docs_snippets/docs_snippets_tests/intro_tutorial_tests/test_cli_invocations.py +++ b/examples/docs_snippets/docs_snippets_tests/intro_tutorial_tests/test_cli_invocations.py @@ -357,6 +357,7 @@ # TODO python command line + # dagit def test_load_repo(): load_dagit_for_workspace_cli_args( diff --git a/js_modules/dagit/public/fonts/fonts.css b/js_modules/dagit/public/fonts/fonts.css --- a/js_modules/dagit/public/fonts/fonts.css +++ b/js_modules/dagit/public/fonts/fonts.css @@ -1,35 +1,40 @@ @font-face { - font-family: 'Open Sans'; + font-family: "Open Sans"; font-style: normal; font-weight: 400; font-display: swap; - src: local('Open Sans Regular'), local('OpenSans-Regular'), url('/fonts/OpenSans-Regular.woff2') format('woff2'); + src: local("Open Sans Regular"), local("OpenSans-Regular"), + url("./OpenSans-Regular.woff2") format("woff2"); } @font-face { - font-family: 'Open Sans'; + font-family: "Open Sans"; font-style: normal; font-weight: 600; font-display: swap; - src: local('Open Sans SemiBold'), local('OpenSans-SemiBold'), url('/fonts/OpenSans-SemiBold.woff2') format('woff2'); + src: local("Open Sans SemiBold"), local("OpenSans-SemiBold"), + url("./OpenSans-SemiBold.woff2") format("woff2"); } @font-face { - font-family: 'Source Code Pro'; + font-family: "Source Code Pro"; font-style: normal; font-weight: 400; font-display: swap; - src: local('Source Code Pro Regular'), local('SourceCodePro-Regular'), url('/fonts/SourceCodePro-Regular.woff2') format('woff2'); + src: local("Source Code Pro Regular"), local("SourceCodePro-Regular"), + url("./SourceCodePro-Regular.woff2") format("woff2"); } @font-face { - font-family: 'Source Code Pro'; + font-family: "Source Code Pro"; font-style: normal; font-weight: 500; font-display: swap; - src: local('Source Code Pro Medium'), local('SourceCodePro-Medium'), url('/fonts/SourceCodePro-Medium.woff2') format('woff2'); + src: local("Source Code Pro Medium"), local("SourceCodePro-Medium"), + url("./SourceCodePro-Medium.woff2") format("woff2"); } @font-face { - font-family: 'Source Code Pro'; + font-family: "Source Code Pro"; font-style: normal; font-weight: 700; font-display: swap; - src: local('Source Code Pro Bold'), local('SourceCodePro-Bold'), url('/fonts/SourceCodePro-Bold.woff2') format('woff2'); + src: local("Source Code Pro Bold"), local("SourceCodePro-Bold"), + url("./SourceCodePro-Bold.woff2") format("woff2"); } diff --git a/js_modules/dagit/public/index.html b/js_modules/dagit/public/index.html --- a/js_modules/dagit/public/index.html +++ b/js_modules/dagit/public/index.html @@ -1,18 +1,20 @@ + + + - - - - + + + - - - - + - - - - Dagit - - - - -
- - - - \ No newline at end of file + + diff --git a/js_modules/dagit/src/App.tsx b/js_modules/dagit/src/App.tsx --- a/js_modules/dagit/src/App.tsx +++ b/js_modules/dagit/src/App.tsx @@ -28,6 +28,7 @@ DagsterRepoOption } from "./DagsterRepositoryContext"; import { CustomTooltipProvider } from "./CustomTooltipProvider"; +import { APP_PATH_PREFIX } from "./DomUtils"; const AppRoutes = () => ( @@ -95,7 +96,7 @@ return (
- + {error ? ( , value: string) { diff --git a/js_modules/dagit/src/execute/LaunchRootExecutionButton.tsx b/js_modules/dagit/src/execute/LaunchRootExecutionButton.tsx --- a/js_modules/dagit/src/execute/LaunchRootExecutionButton.tsx +++ b/js_modules/dagit/src/execute/LaunchRootExecutionButton.tsx @@ -2,7 +2,7 @@ import { useMutation } from "react-apollo"; import { LaunchButton } from "./LaunchButton"; -import { LAUNCH_PIPELINE_EXECUTION_MUTATION, handleExecutionResult } from "../runs/RunUtils"; +import { LAUNCH_PIPELINE_EXECUTION_MUTATION, handleLaunchResult } from "../runs/RunUtils"; import { LaunchPipelineExecutionVariables } from "../runs/types/LaunchPipelineExecution"; interface LaunchRootExecutionButtonProps { @@ -22,7 +22,7 @@ try { const result = await launchPipelineExecution({ variables }); - handleExecutionResult(props.pipelineName, result, { + handleLaunchResult(props.pipelineName, result, { openInNewWindow: true }); } catch (error) { diff --git a/js_modules/dagit/src/runs/Run.tsx b/js_modules/dagit/src/runs/Run.tsx --- a/js_modules/dagit/src/runs/Run.tsx +++ b/js_modules/dagit/src/runs/Run.tsx @@ -11,7 +11,7 @@ import { RunMetadataProvider, IStepState } from "../RunMetadataProvider"; import LogsToolbar from "./LogsToolbar"; import { - handleReexecutionResult, + handleLaunchResult, getReexecutionVariables, LAUNCH_PIPELINE_REEXECUTION_MUTATION } from "./RunUtils"; @@ -245,9 +245,7 @@ repositoryName: repository?.name }); const result = await launchPipelineReexecution({ variables }); - handleReexecutionResult(run.pipeline.name, result, { - openInNewWindow: false - }); + handleLaunchResult(run.pipeline.name, result, { openInNewWindow: false }); }; const onClickStep = (stepKey: string, evt: React.MouseEvent) => { diff --git a/js_modules/dagit/src/runs/RunActionsMenu.tsx b/js_modules/dagit/src/runs/RunActionsMenu.tsx --- a/js_modules/dagit/src/runs/RunActionsMenu.tsx +++ b/js_modules/dagit/src/runs/RunActionsMenu.tsx @@ -25,7 +25,7 @@ CANCEL_MUTATION, DELETE_MUTATION, getReexecutionVariables, - handleReexecutionResult, + handleLaunchResult, RunsQueryRefetchContext } from "./RunUtils"; @@ -98,9 +98,7 @@ repositoryName: repository?.name }) }); - handleReexecutionResult(run.pipelineName, result, { - openInNewWindow: false - }); + handleLaunchResult(run.pipelineName, result, { openInNewWindow: false }); }} /> diff --git a/js_modules/dagit/src/runs/RunUtils.tsx b/js_modules/dagit/src/runs/RunUtils.tsx --- a/js_modules/dagit/src/runs/RunUtils.tsx +++ b/js_modules/dagit/src/runs/RunUtils.tsx @@ -14,6 +14,7 @@ import { Popover, Icon } from "@blueprintjs/core"; import { formatElapsedTime } from "../Util"; +import { APP_PATH_PREFIX } from "../DomUtils"; export function subsetTitleForRun(run: { tags: { key: string; value: string }[] }) { const stepsTag = run.tags.find(t => t.key === "dagster/step_selection"); @@ -28,27 +29,25 @@ refetch: () => void; }>({ refetch: () => {} }); -export function handleExecutionResult( +export function handleLaunchResult( pipelineName: string, - result: void | { - data?: LaunchPipelineExecution; - }, + result: void | { data?: LaunchPipelineExecution | LaunchPipelineReexecution }, opts: { openInNewWindow: boolean } ) { - if (!result || !result.data) { + const obj = + result && result.data && "launchPipelineExecution" in result.data + ? result.data.launchPipelineExecution + : result && result.data && "launchPipelineReexecution" in result.data + ? result.data.launchPipelineReexecution + : null; + + if (!obj) { showCustomAlert({ body: `No data was returned. Did Dagit crash?` }); return; } - const obj = (result.data as LaunchPipelineExecution).launchPipelineExecution; - if (obj.__typename === "LaunchPipelineRunSuccess") { - const url = `/pipeline/${obj.run.pipelineName}/runs/${obj.run.runId}`; - if (opts.openInNewWindow) { - window.open(url, "_blank"); - } else { - window.location.href = url; - } + openRunInBrowser(obj.run, opts); } else if (obj.__typename === "PythonError") { showCustomAlert({ title: "Error", @@ -67,42 +66,15 @@ } } -export function handleReexecutionResult( - pipelineName: string, - result: void | { - data?: LaunchPipelineReexecution; - }, +export function openRunInBrowser( + run: { runId: string; pipelineName: string }, opts: { openInNewWindow: boolean } ) { - if (!result || !result.data) { - showCustomAlert({ body: `No data was returned. Did Dagit crash?` }); - return; - } - - const obj = (result.data as LaunchPipelineReexecution).launchPipelineReexecution; - - if (obj.__typename === "LaunchPipelineRunSuccess") { - const url = `/pipeline/${obj.run.pipeline.name}/runs/${obj.run.runId}`; - if (opts.openInNewWindow) { - window.open(url, "_blank"); - } else { - window.location.href = url; - } - } else if (obj.__typename === "PythonError") { - showCustomAlert({ - title: "Error", - body: - }); + const url = `${APP_PATH_PREFIX}/pipeline/${run.pipelineName}/runs/${run.runId}`; + if (opts.openInNewWindow) { + window.open(url, "_blank"); } else { - let message = `${pipelineName} cannot be executed with the provided config.`; - - if ("errors" in obj) { - message += ` Please fix the following errors:\n\n${obj.errors - .map(error => error.message) - .join("\n\n")}`; - } - - showCustomAlert({ body: message }); + window.location.href = url; } } @@ -296,9 +268,7 @@ ... on LaunchPipelineRunSuccess { run { runId - pipeline { - name - } + pipelineName rootRunId parentRunId } diff --git a/js_modules/dagit/src/runs/types/LaunchPipelineReexecution.ts b/js_modules/dagit/src/runs/types/LaunchPipelineReexecution.ts --- a/js_modules/dagit/src/runs/types/LaunchPipelineReexecution.ts +++ b/js_modules/dagit/src/runs/types/LaunchPipelineReexecution.ts @@ -13,15 +13,10 @@ __typename: "InvalidStepError" | "InvalidOutputError" | "PipelineRunConflict" | "PresetNotFoundError" | "ConflictingExecutionParamsError"; } -export interface LaunchPipelineReexecution_launchPipelineReexecution_LaunchPipelineRunSuccess_run_pipeline { - __typename: "PipelineSnapshot" | "UnknownPipeline"; - name: string; -} - export interface LaunchPipelineReexecution_launchPipelineReexecution_LaunchPipelineRunSuccess_run { __typename: "PipelineRun"; runId: string; - pipeline: LaunchPipelineReexecution_launchPipelineReexecution_LaunchPipelineRunSuccess_run_pipeline; + pipelineName: string; rootRunId: string | null; parentRunId: string | null; } diff --git a/js_modules/dagit/src/schedules/PartitionTable.tsx b/js_modules/dagit/src/schedules/PartitionTable.tsx --- a/js_modules/dagit/src/schedules/PartitionTable.tsx +++ b/js_modules/dagit/src/schedules/PartitionTable.tsx @@ -1,11 +1,11 @@ import * as React from "react"; +import { createGlobalStyle } from "styled-components/macro"; +import { Line } from "react-chartjs-2"; import { PartitionLongitudinalQuery_partitionSetOrError_PartitionSet_partitionsOrError_Partitions_results_runs } from "./types/PartitionLongitudinalQuery"; import { RowContainer } from "../ListComponents"; - -import { createGlobalStyle } from "styled-components/macro"; -import { Line } from "react-chartjs-2"; import { RUN_STATUS_COLORS, RUN_STATUS_HOVER_COLORS } from "../runs/RunStatusDots"; +import { openRunInBrowser } from "../runs/RunUtils"; type Run = PartitionLongitudinalQuery_partitionSetOrError_PartitionSet_partitionsOrError_Partitions_results_runs; interface Point { @@ -68,7 +68,7 @@ } const point = datasets[element._datasetIndex].data[element._index]; if (point && point.runId) { - window.open(`/pipeline/${point.pipelineName}/runs/${point.runId}`, "_blank"); + openRunInBrowser(point, { openInNewWindow: true }); } }; diff --git a/python_modules/dagit/dagit/app.py b/python_modules/dagit/dagit/app.py --- a/python_modules/dagit/dagit/app.py +++ b/python_modules/dagit/dagit/app.py @@ -8,7 +8,7 @@ from dagster_graphql.implementation.context import DagsterGraphQLContext from dagster_graphql.schema import create_schema from dagster_graphql.version import __version__ as dagster_graphql_version -from flask import Flask, jsonify, redirect, request, send_file +from flask import Blueprint, Flask, jsonify, redirect, request, send_file from flask_cors import CORS from flask_graphql import GraphQLView from flask_sockets import Sockets @@ -74,19 +74,6 @@ ) -def index_view(_path): - try: - return send_file(os.path.join(os.path.dirname(__file__), './webapp/build/index.html')) - except seven.FileNotFoundError: - text = '''

Can't find webapp files. Probably webapp isn't built. If you are using - dagit, then probably it's a corrupted installation or a bug. However, if you are - developing dagit locally, your problem can be fixed as follows:

- -
cd ./python_modules/
-make rebuild_dagit
''' - return text, 500 - - def notebook_view(request_args): check.dict_param(request_args, 'request_args') @@ -134,37 +121,42 @@ return view -def instantiate_app_with_views(context): +def instantiate_app_with_views(context, app_path_prefix): app = Flask( 'dagster-ui', - static_url_path='', + static_url_path=app_path_prefix, static_folder=os.path.join(os.path.dirname(__file__), './webapp/build'), ) - sockets = Sockets(app) - app.app_protocol = lambda environ_path_info: 'graphql-ws' - schema = create_schema() subscription_server = DagsterSubscriptionServer(schema=schema) - app.add_url_rule( + # Websocket routes + sockets = Sockets(app) + sockets.add_url_rule( + '{}/graphql'.format(app_path_prefix), + 'graphql', + dagster_graphql_subscription_view(subscription_server, context), + ) + + # HTTP routes + bp = Blueprint('routes', __name__, url_prefix=app_path_prefix) + bp.add_url_rule( + '/graphiql', 'graphiql', lambda: redirect('{}/graphql'.format(app_path_prefix), 301) + ) + bp.add_url_rule( '/graphql', 'graphql', DagsterGraphQLView.as_view( 'graphql', schema=schema, graphiql=True, - # XXX(freiksenet): Pass proper ws url - graphiql_template=PLAYGROUND_TEMPLATE, + graphiql_template=PLAYGROUND_TEMPLATE.replace('APP_PATH_PREFIX', app_path_prefix), executor=Executor(), context=context, ), ) - app.add_url_rule('/graphiql', 'graphiql', lambda: redirect('/graphql', 301)) - sockets.add_url_rule( - '/graphql', 'graphql', dagster_graphql_subscription_view(subscription_server, context) - ) - app.add_url_rule( + bp.add_url_rule( # should match the `build_local_download_url` '/download///', 'download_view', @@ -174,17 +166,56 @@ # these routes are specifically for the Dagit UI and are not part of the graphql # API that we want other people to consume, so they're separate for now. # Also grabbing the magic global request args dict so that notebook_view is testable - app.add_url_rule('/dagit/notebook', 'notebook', lambda: notebook_view(request.args)) + bp.add_url_rule('/dagit/notebook', 'notebook', lambda: notebook_view(request.args)) + bp.add_url_rule('/dagit_info', 'sanity_view', info_view) + + index_path = os.path.join(os.path.dirname(__file__), './webapp/build/index.html') + + def index_view(_path): + try: + with open(index_path) as f: + return ( + f.read() + .replace('href="/', 'href="{}/'.format(app_path_prefix)) + .replace('src="/', 'src="{}/'.format(app_path_prefix)) + .replace( + ' GraphQL Playground - - - + + + diff --git a/python_modules/dagit/dagit_tests/test_app.py b/python_modules/dagit/dagit_tests/test_app.py --- a/python_modules/dagit/dagit_tests/test_app.py +++ b/python_modules/dagit/dagit_tests/test_app.py @@ -58,12 +58,44 @@ assert b'You need to enable JavaScript to run this app' in res.data +def test_index_view_at_path_prefix(): + workspace = load_workspace_from_yaml_path(file_relative_path(__file__, './workspace.yaml')) + with create_app_from_workspace( + workspace, DagsterInstance.ephemeral(), '/dagster-path' + ).test_client() as client: + # / redirects to prefixed path + res = client.get('/') + assert res.status_code == 301 + + # index contains the path meta tag + res = client.get('/dagster-path') + assert res.status_code == 200 + assert b'You need to enable JavaScript to run this app' in res.data + assert b'