diff --git a/js_modules/dagit/src/__tests__/__snapshots__/App.test.tsx.snap b/js_modules/dagit/src/__tests__/__snapshots__/App.test.tsx.snap --- a/js_modules/dagit/src/__tests__/__snapshots__/App.test.tsx.snap +++ b/js_modules/dagit/src/__tests__/__snapshots__/App.test.tsx.snap @@ -94,21 +94,21 @@ className="bp3-navbar-group bp3-align-right" > Runs Schedule @@ -266,21 +266,21 @@ className="bp3-navbar-group bp3-align-right" > Runs Schedule @@ -410,17 +410,17 @@ className="bp3-navbar-divider" /> Explore @@ -433,21 +433,21 @@ className="bp3-navbar-group bp3-align-right" > Runs Schedule @@ -605,17 +605,17 @@ className="bp3-navbar-divider" /> Explore @@ -628,21 +628,21 @@ className="bp3-navbar-group bp3-align-right" > Runs Schedule @@ -800,17 +800,17 @@ className="bp3-navbar-divider" /> Explore @@ -823,21 +823,21 @@ className="bp3-navbar-group bp3-align-right" > Runs Schedule @@ -995,17 +995,17 @@ className="bp3-navbar-divider" /> Explore @@ -1018,21 +1018,21 @@ className="bp3-navbar-group bp3-align-right" > Runs Schedule @@ -1190,21 +1190,21 @@ className="bp3-navbar-group bp3-align-right" > Runs Schedule diff --git a/js_modules/dagit/src/schedules/ScheduleRoot.tsx b/js_modules/dagit/src/schedules/ScheduleRoot.tsx --- a/js_modules/dagit/src/schedules/ScheduleRoot.tsx +++ b/js_modules/dagit/src/schedules/ScheduleRoot.tsx @@ -1,8 +1,11 @@ import * as React from "react"; -import styled from "styled-components"; -import { Colors } from "@blueprintjs/core"; -import { Header, ScrollContainer } from "../ListComponents"; +import { + Header, + ScrollContainer, + RowColumn, + RowContainer +} from "../ListComponents"; import { Query, QueryResult } from "react-apollo"; import { ScheduleRootQuery, @@ -11,12 +14,13 @@ import Loading from "../Loading"; import gql from "graphql-tag"; import { match } from "react-router"; -import ScheduleRow, { ScheduleRowFragment } from "./ScheduleRow"; +import { Link } from "react-router-dom"; +import { ScheduleRow, ScheduleRowFragment, AttemptStatus } from "./ScheduleRow"; import { HighlightedCodeBlock } from "../HighlightedCodeBlock"; import { showCustomAlert } from "../CustomAlertProvider"; import { unixTimestampToString } from "../Util"; -import { ScheduleAttemptStatus } from "../types/globalTypes"; +import { RunStatus } from "../runs/RunUtils"; const NUM_RUNS_TO_DISPLAY = 10; const NUM_ATTEMPTS_TO_DISPLAY = 25; @@ -72,81 +76,72 @@ const AttemptsTable: React.FunctionComponent = ({ attemptList }) => { + if (!attemptList || !attemptList.length) { + return null; + } return ( <> Attempts {attemptList.map((attempt, i) => ( - - - - {unixTimestampToString(attempt.time)} - + + + {attempt.run ? ( + + ) : ( + + )} + + + {attempt.run ? ( + + + {attempt.run.runId} + + + ) : ( + + + showCustomAlert({ + title: "Schedule Response", + body: ( + <> + + > + ) + }) + } + > + {" "} + View error + + + )} + + + {unixTimestampToString(attempt.time)} + + ))} > ); }; -export const Link = styled.a` - color: ${Colors.DARK_GRAY4}; - text-decoration: underline; - margin-left: 10px; -`; - -const AttemptMessage = ({ result }: { result: string }) => { - return ( - - - showCustomAlert({ - body: ( - <> - - > - ) - }) - } - > - View Response - - - ); -}; - -const AttemptStatus = styled.div<{ status: ScheduleAttemptStatus }>` - display: inline-block; - width: 11px; - height: 11px; - border-radius: 5.5px; - align-self: center; - transition: background 200ms linear; - background: ${({ status }) => - ({ - [ScheduleAttemptStatus.SUCCESS]: Colors.GREEN2, - [ScheduleAttemptStatus.ERROR]: Colors.RED3, - [ScheduleAttemptStatus.SKIPPED]: Colors.GOLD3 - }[status])}; - &:hover { - background: ${({ status }) => - ({ - [ScheduleAttemptStatus.SUCCESS]: Colors.GREEN2, - [ScheduleAttemptStatus.ERROR]: Colors.RED3, - [ScheduleAttemptStatus.SKIPPED]: Colors.GOLD3 - }[status])}; - } -`; - export const SCHEDULE_ROOT_QUERY = gql` query ScheduleRootQuery( $scheduleName: String! @@ -164,6 +159,10 @@ time jsonResult status + run { + runId + status + } } } ... on ScheduleNotFoundError { diff --git a/js_modules/dagit/src/schedules/ScheduleRow.tsx b/js_modules/dagit/src/schedules/ScheduleRow.tsx --- a/js_modules/dagit/src/schedules/ScheduleRow.tsx +++ b/js_modules/dagit/src/schedules/ScheduleRow.tsx @@ -5,6 +5,7 @@ import { Button, Classes, + Colors, Switch, Icon, Menu, @@ -18,12 +19,9 @@ import { HighlightedCodeBlock } from "../HighlightedCodeBlock"; import { RowColumn, RowContainer } from "../ListComponents"; import { RunStatus, titleForRun } from "../runs/RunUtils"; -import { - ScheduleFragment, - ScheduleFragment_runs -} from "./types/ScheduleFragment"; +import { ScheduleFragment } from "./types/ScheduleFragment"; import { ScheduleStatus, ScheduleAttemptStatus } from "../types/globalTypes"; -import { Link } from "react-router-dom"; +import { Link, useRouteMatch } from "react-router-dom"; import cronstrue from "cronstrue"; import gql from "graphql-tag"; import { showCustomAlert } from "../CustomAlertProvider"; @@ -32,17 +30,16 @@ const NUM_RUNS_TO_DISPLAY = 10; -const ScheduleRow: React.FunctionComponent<{ +export const ScheduleRow: React.FunctionComponent<{ schedule: ScheduleFragment; }> = ({ schedule }) => { const { id, status, scheduleDefinition, - runs, - runsCount, logsPath, - attempts + attempts, + attemptsCount } = schedule; const { name, @@ -56,6 +53,7 @@ const [startSchedule] = useMutation(START_SCHEDULE_MUTATION); const [stopSchedule] = useMutation(STOP_SCHEDULE_MUTATION); + const match = useRouteMatch("/schedule/:scheduleName"); const mostRecentAttempt = attempts.length > 0 ? attempts[0] : null; const mostRecentAttemptLogError = mostRecentAttempt @@ -70,18 +68,13 @@ } }; - const sortRuns = (runs: ScheduleFragment_runs[]) => { - if (!runs) return []; - - return runs.sort((a, b) => { - const aStart = a.stats.startTime || Number.MAX_SAFE_INTEGER; - const bStart = b.stats.startTime || Number.MAX_SAFE_INTEGER; - return bStart - aStart; - }); - }; - - const sortedRuns = sortRuns(runs); - const mostRecentRun = sortedRuns[0]; + const displayName = match ? ( + {name} + ) : ( + + {name} + + ); return ( @@ -124,9 +117,7 @@ }} /> - - {name} - + {displayName} {pipelineName} @@ -150,45 +141,78 @@ )} - {runs && runs.length > 0 - ? runs.slice(0, NUM_RUNS_TO_DISPLAY).map(run => ( + {attempts && attempts.length > 0 + ? attempts.slice(0, NUM_RUNS_TO_DISPLAY).map((attempt, i) => ( - - + + + + + ) : ( + + showCustomAlert({ + title: "Schedule Response", + body: ( + <> + + > + ) + }) + } > - - - + + + + + )} )) : "-"} - {runsCount > NUM_RUNS_TO_DISPLAY && ( + {attemptsCount > NUM_RUNS_TO_DISPLAY && ( {" "} - +{runsCount - NUM_RUNS_TO_DISPLAY} more + +{attemptsCount - NUM_RUNS_TO_DISPLAY} more )} - {mostRecentRun - ? unixTimestampToString(mostRecentRun.stats.startTime) - : "No previous runs"} + {mostRecentAttempt + ? unixTimestampToString(mostRecentAttempt.time) + : "-"} {mostRecentAttempt && mostRecentAttempt.status === ScheduleAttemptStatus.ERROR && ( @@ -292,22 +316,19 @@ cronSchedule } logsPath - runsCount - attempts(limit: 1) { + attempts(limit: $limit) { + run { + runId + pipeline { + name + } + status + } time jsonResult status } - runs(limit: $limit) { - runId - pipeline { - name - } - status - stats { - startTime - } - } + attemptsCount status } `; @@ -363,4 +384,25 @@ } `; -export default ScheduleRow; +export const AttemptStatus = styled.div<{ status: ScheduleAttemptStatus }>` + display: inline-block; + width: 11px; + height: 11px; + border-radius: 5.5px; + align-self: center; + transition: background 200ms linear; + background: ${({ status }) => + ({ + [ScheduleAttemptStatus.SUCCESS]: Colors.GREEN2, + [ScheduleAttemptStatus.ERROR]: Colors.RED3, + [ScheduleAttemptStatus.SKIPPED]: Colors.GOLD3 + }[status])}; + &:hover { + background: ${({ status }) => + ({ + [ScheduleAttemptStatus.SUCCESS]: Colors.GREEN2, + [ScheduleAttemptStatus.ERROR]: Colors.RED3, + [ScheduleAttemptStatus.SKIPPED]: Colors.GOLD3 + }[status])}; + } +`; diff --git a/js_modules/dagit/src/schedules/SchedulesRoot.tsx b/js_modules/dagit/src/schedules/SchedulesRoot.tsx --- a/js_modules/dagit/src/schedules/SchedulesRoot.tsx +++ b/js_modules/dagit/src/schedules/SchedulesRoot.tsx @@ -15,7 +15,7 @@ import Loading from "../Loading"; import gql from "graphql-tag"; -import ScheduleRow, { ScheduleRowFragment } from "./ScheduleRow"; +import { ScheduleRow, ScheduleRowFragment } from "./ScheduleRow"; const NUM_RUNS_TO_DISPLAY = 10; @@ -113,8 +113,8 @@ Schedule Name Pipeline Schedule - Recent Runs - Last Run + Recent Attempts + Last Attempt Execution Params )} diff --git a/js_modules/dagit/src/schedules/types/ScheduleFragment.ts b/js_modules/dagit/src/schedules/types/ScheduleFragment.ts --- a/js_modules/dagit/src/schedules/types/ScheduleFragment.ts +++ b/js_modules/dagit/src/schedules/types/ScheduleFragment.ts @@ -3,7 +3,7 @@ /* eslint-disable */ // This file was automatically generated and should not be edited. -import { ScheduleAttemptStatus, PipelineRunStatus, ScheduleStatus } from "./../../types/globalTypes"; +import { PipelineRunStatus, ScheduleAttemptStatus, ScheduleStatus } from "./../../types/globalTypes"; // ==================================================== // GraphQL fragment: ScheduleFragment @@ -17,29 +17,24 @@ cronSchedule: string; } -export interface ScheduleFragment_attempts { - __typename: "ScheduleAttempt"; - time: number; - jsonResult: string; - status: ScheduleAttemptStatus; -} - -export interface ScheduleFragment_runs_pipeline { +export interface ScheduleFragment_attempts_run_pipeline { __typename: "Pipeline" | "UnknownPipeline"; name: string; } -export interface ScheduleFragment_runs_stats { - __typename: "PipelineRunStatsSnapshot"; - startTime: number | null; -} - -export interface ScheduleFragment_runs { +export interface ScheduleFragment_attempts_run { __typename: "PipelineRun"; runId: string; - pipeline: ScheduleFragment_runs_pipeline; + pipeline: ScheduleFragment_attempts_run_pipeline; status: PipelineRunStatus; - stats: ScheduleFragment_runs_stats; +} + +export interface ScheduleFragment_attempts { + __typename: "ScheduleAttempt"; + run: ScheduleFragment_attempts_run | null; + time: number; + jsonResult: string; + status: ScheduleAttemptStatus; } export interface ScheduleFragment { @@ -47,8 +42,7 @@ id: string; scheduleDefinition: ScheduleFragment_scheduleDefinition; logsPath: string; - runsCount: number; attempts: ScheduleFragment_attempts[]; - runs: ScheduleFragment_runs[]; + attemptsCount: number; status: ScheduleStatus; } diff --git a/js_modules/dagit/src/schedules/types/ScheduleRootQuery.ts b/js_modules/dagit/src/schedules/types/ScheduleRootQuery.ts --- a/js_modules/dagit/src/schedules/types/ScheduleRootQuery.ts +++ b/js_modules/dagit/src/schedules/types/ScheduleRootQuery.ts @@ -3,7 +3,7 @@ /* eslint-disable */ // This file was automatically generated and should not be edited. -import { ScheduleAttemptStatus, PipelineRunStatus, ScheduleStatus } from "./../../types/globalTypes"; +import { PipelineRunStatus, ScheduleAttemptStatus, ScheduleStatus } from "./../../types/globalTypes"; // ==================================================== // GraphQL query operation: ScheduleRootQuery @@ -17,29 +17,30 @@ cronSchedule: string; } +export interface ScheduleRootQuery_scheduleOrError_RunningSchedule_attempts_run_pipeline { + __typename: "Pipeline" | "UnknownPipeline"; + name: string; +} + +export interface ScheduleRootQuery_scheduleOrError_RunningSchedule_attempts_run { + __typename: "PipelineRun"; + runId: string; + pipeline: ScheduleRootQuery_scheduleOrError_RunningSchedule_attempts_run_pipeline; + status: PipelineRunStatus; +} + export interface ScheduleRootQuery_scheduleOrError_RunningSchedule_attempts { __typename: "ScheduleAttempt"; + run: ScheduleRootQuery_scheduleOrError_RunningSchedule_attempts_run | null; time: number; jsonResult: string; status: ScheduleAttemptStatus; } -export interface ScheduleRootQuery_scheduleOrError_RunningSchedule_runs_pipeline { - __typename: "Pipeline" | "UnknownPipeline"; - name: string; -} - -export interface ScheduleRootQuery_scheduleOrError_RunningSchedule_runs_stats { - __typename: "PipelineRunStatsSnapshot"; - startTime: number | null; -} - -export interface ScheduleRootQuery_scheduleOrError_RunningSchedule_runs { +export interface ScheduleRootQuery_scheduleOrError_RunningSchedule_attemptList_run { __typename: "PipelineRun"; runId: string; - pipeline: ScheduleRootQuery_scheduleOrError_RunningSchedule_runs_pipeline; status: PipelineRunStatus; - stats: ScheduleRootQuery_scheduleOrError_RunningSchedule_runs_stats; } export interface ScheduleRootQuery_scheduleOrError_RunningSchedule_attemptList { @@ -47,6 +48,7 @@ time: number; jsonResult: string; status: ScheduleAttemptStatus; + run: ScheduleRootQuery_scheduleOrError_RunningSchedule_attemptList_run | null; } export interface ScheduleRootQuery_scheduleOrError_RunningSchedule { @@ -54,9 +56,8 @@ id: string; scheduleDefinition: ScheduleRootQuery_scheduleOrError_RunningSchedule_scheduleDefinition; logsPath: string; - runsCount: number; attempts: ScheduleRootQuery_scheduleOrError_RunningSchedule_attempts[]; - runs: ScheduleRootQuery_scheduleOrError_RunningSchedule_runs[]; + attemptsCount: number; status: ScheduleStatus; attemptList: ScheduleRootQuery_scheduleOrError_RunningSchedule_attemptList[]; } diff --git a/js_modules/dagit/src/schedules/types/SchedulesRootQuery.ts b/js_modules/dagit/src/schedules/types/SchedulesRootQuery.ts --- a/js_modules/dagit/src/schedules/types/SchedulesRootQuery.ts +++ b/js_modules/dagit/src/schedules/types/SchedulesRootQuery.ts @@ -3,7 +3,7 @@ /* eslint-disable */ // This file was automatically generated and should not be edited. -import { ScheduleAttemptStatus, PipelineRunStatus, ScheduleStatus } from "./../../types/globalTypes"; +import { PipelineRunStatus, ScheduleAttemptStatus, ScheduleStatus } from "./../../types/globalTypes"; // ==================================================== // GraphQL query operation: SchedulesRootQuery @@ -22,29 +22,24 @@ cronSchedule: string; } -export interface SchedulesRootQuery_scheduler_Scheduler_runningSchedules_attempts { - __typename: "ScheduleAttempt"; - time: number; - jsonResult: string; - status: ScheduleAttemptStatus; -} - -export interface SchedulesRootQuery_scheduler_Scheduler_runningSchedules_runs_pipeline { +export interface SchedulesRootQuery_scheduler_Scheduler_runningSchedules_attempts_run_pipeline { __typename: "Pipeline" | "UnknownPipeline"; name: string; } -export interface SchedulesRootQuery_scheduler_Scheduler_runningSchedules_runs_stats { - __typename: "PipelineRunStatsSnapshot"; - startTime: number | null; -} - -export interface SchedulesRootQuery_scheduler_Scheduler_runningSchedules_runs { +export interface SchedulesRootQuery_scheduler_Scheduler_runningSchedules_attempts_run { __typename: "PipelineRun"; runId: string; - pipeline: SchedulesRootQuery_scheduler_Scheduler_runningSchedules_runs_pipeline; + pipeline: SchedulesRootQuery_scheduler_Scheduler_runningSchedules_attempts_run_pipeline; status: PipelineRunStatus; - stats: SchedulesRootQuery_scheduler_Scheduler_runningSchedules_runs_stats; +} + +export interface SchedulesRootQuery_scheduler_Scheduler_runningSchedules_attempts { + __typename: "ScheduleAttempt"; + run: SchedulesRootQuery_scheduler_Scheduler_runningSchedules_attempts_run | null; + time: number; + jsonResult: string; + status: ScheduleAttemptStatus; } export interface SchedulesRootQuery_scheduler_Scheduler_runningSchedules { @@ -52,9 +47,8 @@ id: string; scheduleDefinition: SchedulesRootQuery_scheduler_Scheduler_runningSchedules_scheduleDefinition; logsPath: string; - runsCount: number; attempts: SchedulesRootQuery_scheduler_Scheduler_runningSchedules_attempts[]; - runs: SchedulesRootQuery_scheduler_Scheduler_runningSchedules_runs[]; + attemptsCount: number; status: ScheduleStatus; } diff --git a/js_modules/dagit/src/schema.graphql b/js_modules/dagit/src/schema.graphql --- a/js_modules/dagit/src/schema.graphql +++ b/js_modules/dagit/src/schema.graphql @@ -884,6 +884,7 @@ runs(limit: Int): [PipelineRun!]! runsCount: Int! attempts(limit: Int): [ScheduleAttempt!]! + attemptsCount: Int! logsPath: String! } diff --git a/python_modules/dagster-graphql/dagster_graphql/implementation/fetch_schedules.py b/python_modules/dagster-graphql/dagster_graphql/implementation/fetch_schedules.py --- a/python_modules/dagster-graphql/dagster_graphql/implementation/fetch_schedules.py +++ b/python_modules/dagster-graphql/dagster_graphql/implementation/fetch_schedules.py @@ -1,3 +1,6 @@ +import glob +import os + from graphql.execution.base import ResolveInfo from dagster import check @@ -44,3 +47,9 @@ ) return schedule + + +def get_schedule_attempt_filenames(graphene_info, schedule_name): + scheduler = graphene_info.context.get_scheduler() + log_dir = scheduler.log_path_for_schedule(schedule_name) + return glob.glob(os.path.join(log_dir, "*.result")) diff --git a/python_modules/dagster-graphql/dagster_graphql/schema/run_schedule.py b/python_modules/dagster-graphql/dagster_graphql/schema/run_schedule.py --- a/python_modules/dagster-graphql/dagster_graphql/schema/run_schedule.py +++ b/python_modules/dagster-graphql/dagster_graphql/schema/run_schedule.py @@ -1,11 +1,13 @@ -import glob import heapq import json import os import yaml from dagster_graphql import dauphin -from dagster_graphql.implementation.fetch_schedules import get_dagster_schedule_def +from dagster_graphql.implementation.fetch_schedules import ( + get_dagster_schedule_def, + get_schedule_attempt_filenames, +) from dagster_graphql.implementation.utils import UserFacingGraphQLError, capture_dauphin_error from dagster_graphql.schema.errors import ( DauphinScheduleNotFoundError, @@ -129,6 +131,7 @@ runs = dauphin.Field(dauphin.non_null_list('PipelineRun'), limit=dauphin.Int()) runs_count = dauphin.NonNull(dauphin.Int) attempts = dauphin.Field(dauphin.non_null_list('ScheduleAttempt'), limit=dauphin.Int()) + attempts_count = dauphin.NonNull(dauphin.Int) logs_path = dauphin.NonNull(dauphin.String) def __init__(self, graphene_info, schedule): @@ -147,10 +150,7 @@ def resolve_attempts(self, graphene_info, **kwargs): limit = kwargs.get('limit') - scheduler = graphene_info.context.get_scheduler() - log_dir = scheduler.log_path_for_schedule(self._schedule.name) - - results = glob.glob(os.path.join(log_dir, "*.result")) + results = get_schedule_attempt_filenames(graphene_info, self._schedule.name) if limit is None: limit = len(results) latest_results = heapq.nlargest(limit, results, key=os.path.getctime) @@ -196,6 +196,10 @@ return attempts + def resolve_attempts_count(self, graphene_info): + attempt_files = get_schedule_attempt_filenames(graphene_info, self._schedule.name) + return len(attempt_files) + def resolve_logs_path(self, graphene_info): scheduler = graphene_info.context.get_scheduler() return scheduler.log_path_for_schedule(self._schedule.name)