diff --git a/js_modules/dagit/src/CursorControls.tsx b/js_modules/dagit/src/CursorControls.tsx new file mode 100644 --- /dev/null +++ b/js_modules/dagit/src/CursorControls.tsx @@ -0,0 +1,71 @@ +import {Button} from '@blueprintjs/core'; +import {IconNames} from '@blueprintjs/icons'; +import * as React from 'react'; + +export interface CursorPaginationProps { + hasPrevCursor: boolean; + hasNextCursor: boolean; + popCursor: () => void; + advanceCursor: () => void; + reset: () => void; +} + +export const CursorPaginationControls: React.FunctionComponent = ({ + hasPrevCursor, + hasNextCursor, + popCursor, + advanceCursor, +}) => { + return ( +
+ + +
+ ); +}; + +export const CursorHistoryControls: React.FunctionComponent = ({ + hasPrevCursor, + hasNextCursor, + popCursor, + advanceCursor, +}) => { + return ( +
+ + +
+ ); +}; diff --git a/js_modules/dagit/src/CursorPaginationControls.tsx b/js_modules/dagit/src/CursorPaginationControls.tsx deleted file mode 100644 --- a/js_modules/dagit/src/CursorPaginationControls.tsx +++ /dev/null @@ -1,37 +0,0 @@ -import {Button} from '@blueprintjs/core'; -import {IconNames} from '@blueprintjs/icons'; -import * as React from 'react'; - -import {CursorPaginationProps} from 'src/runs/useCursorPaginatedQuery'; - -export const CursorPaginationControls: React.FunctionComponent = ({ - hasPrevPage, - hasNextPage, - onPrevPage, - onNextPage, -}) => { - return ( -
- - -
- ); -}; diff --git a/js_modules/dagit/src/PipelineRunsRoot.tsx b/js_modules/dagit/src/PipelineRunsRoot.tsx --- a/js_modules/dagit/src/PipelineRunsRoot.tsx +++ b/js_modules/dagit/src/PipelineRunsRoot.tsx @@ -5,7 +5,7 @@ import {RouteComponentProps} from 'react-router'; import styled from 'styled-components/macro'; -import {CursorPaginationControls} from 'src/CursorPaginationControls'; +import {CursorPaginationControls} from 'src/CursorControls'; import {ScrollContainer} from 'src/ListComponents'; import {Loading} from 'src/Loading'; import {explorerPathFromString} from 'src/PipelinePathUtils'; diff --git a/js_modules/dagit/src/partitions/PartitionPageControls.tsx b/js_modules/dagit/src/partitions/PartitionPageControls.tsx new file mode 100644 --- /dev/null +++ b/js_modules/dagit/src/partitions/PartitionPageControls.tsx @@ -0,0 +1,51 @@ +import {Button, ButtonGroup} from '@blueprintjs/core'; +import * as React from 'react'; +import styled from 'styled-components/macro'; + +import {CursorHistoryControls, CursorPaginationProps} from 'src/CursorControls'; + +interface PartitionPageControlsProps { + pageSize: number | undefined; + paginationProps: CursorPaginationProps; + setPageSize: React.Dispatch>; +} + +export const PartitionPageControls: React.FunctionComponent = ({ + pageSize, + children, + setPageSize, + paginationProps, +}) => ( + + + + {[7, 30, 120].map((size) => ( + + ))} + + + {children} + + + +); + +const PartitionPagerContainer = styled.div` + display: flex; + align-items: center; + justify-content: space-between; + margin: 10px 0; +`; + +const PartitionPagerLeftContainer = styled.div` + display: flex; + align-items: center; +`; diff --git a/js_modules/dagit/src/partitions/PartitionView.tsx b/js_modules/dagit/src/partitions/PartitionView.tsx --- a/js_modules/dagit/src/partitions/PartitionView.tsx +++ b/js_modules/dagit/src/partitions/PartitionView.tsx @@ -1,25 +1,19 @@ -import {Button, ButtonGroup, Divider, Spinner} from '@blueprintjs/core'; +import {Divider, Spinner} from '@blueprintjs/core'; import {Colors} from '@blueprintjs/core'; -import {IconNames} from '@blueprintjs/icons'; -import gql from 'graphql-tag'; import * as React from 'react'; -import {Query, QueryResult} from 'react-apollo'; import styled from 'styled-components/macro'; -import {useRepositorySelector} from 'src/DagsterRepositoryContext'; import {Header} from 'src/ListComponents'; -import {Loading} from 'src/Loading'; -import {PythonErrorInfo} from 'src/PythonErrorInfo'; import {TokenizingFieldValue} from 'src/TokenizingField'; import {colorHash} from 'src/Util'; import {PIPELINE_LABEL, PartitionGraph} from 'src/partitions/PartitionGraph'; +import {PartitionPageControls} from 'src/partitions/PartitionPageControls'; import {PartitionRunMatrix} from 'src/partitions/PartitionRunMatrix'; import { - PartitionLongitudinalQuery, PartitionLongitudinalQuery_partitionSetOrError_PartitionSet_partitionsOrError_Partitions_results, PartitionLongitudinalQuery_partitionSetOrError_PartitionSet_partitionsOrError_Partitions_results_runs, } from 'src/partitions/types/PartitionLongitudinalQuery'; -import {RunTable} from 'src/runs/RunTable'; +import {useChunkedPartitionsQuery} from 'src/partitions/useChunkedPartitionsQuery'; import {RunsFilter} from 'src/runs/RunsFilter'; type Partition = PartitionLongitudinalQuery_partitionSetOrError_PartitionSet_partitionsOrError_Partitions_results; @@ -28,109 +22,57 @@ interface PartitionViewProps { pipelineName: string; partitionSetName: string; - cursor: string | undefined; - setCursor: (cursor: string | undefined) => void; - onLoaded?: () => void; runTags?: {[key: string]: string}; } export const PartitionView: React.FunctionComponent = ({ pipelineName, partitionSetName, - cursor, - setCursor, - onLoaded, runTags, }) => { - const [cursorStack, setCursorStack] = React.useState([]); - const [pageSize, setPageSize] = React.useState(30); - const repositorySelector = useRepositorySelector(); - const popCursor = () => { - const nextStack = [...cursorStack]; - setCursor(nextStack.pop()); - setCursorStack(nextStack); - }; - const pushCursor = (nextCursor: string) => { - if (cursor) { - setCursorStack([...cursorStack, cursor]); - } - setCursor(nextCursor); - }; + const [pageSize, setPageSize] = React.useState(30); + const {loading, partitions, paginationProps} = useChunkedPartitionsQuery( + partitionSetName, + pageSize, + ); + + const allStepKeys = {}; + partitions.forEach((partition) => { + partition.runs?.forEach((run) => { + if (!run) { + return; + } + run.stepStats.forEach((stat) => { + allStepKeys[stat.stepKey] = true; + }); + }); + }); return ( - - {(queryResult: QueryResult) => ( - - {({partitionSetOrError}) => { - onLoaded?.(); - if (partitionSetOrError.__typename !== 'PartitionSet') { - return null; - } - const partitionSet = partitionSetOrError; - const partitions = - partitionSet.partitionsOrError.__typename === 'Partitions' - ? partitionSet.partitionsOrError.results - : []; - const allStepKeys = {}; - partitions.forEach((partition) => { - partition.runs?.forEach((run) => { - if (!run) { - return; - } - run.stepStats.forEach((stat) => { - allStepKeys[stat.stepKey] = true; - }); - }); - }); - const showLoading = queryResult.loading && queryResult.networkStatus !== 6; - return ( -
-
Longitudinal History
- - -
- - - {showLoading && ( - - - - )} -
-
- ); - }} - - )} - +
+
Longitudinal History
+ + { + setPageSize(next); + paginationProps.reset(); + }} + > + {loading && ( +
+ +
+ Loading Partitions... +
+ )} + +
+ + +
+
); }; @@ -228,13 +170,10 @@ ); }; -const StepSelector = ({ - selected, - onChange, -}: { +const StepSelector: React.FunctionComponent<{ selected: {[stepKey: string]: boolean}; onChange: (selected: {[stepKey: string]: boolean}) => void; -}) => { +}> = ({selected, onChange}) => { const onStepClick = (stepKey: string) => { return (evt: React.MouseEvent) => { if (evt.shiftKey) { @@ -329,20 +268,6 @@ margin: 0 auto; `; -const Overlay = styled.div` - position: absolute; - top: 0; - right: 0; - bottom: 0; - left: 0; - background-color: #ffffff; - opacity: 0.8; - display: flex; - justify-content: center; - align-items: center; - z-index: 1000; -`; - const getPipelineDurationForRun = (run: Run) => { const {stats} = run; if ( @@ -462,138 +387,3 @@ } return true; }; - -interface PartitionPagerProps { - displayed: Partition[]; - pageSize: number | undefined; - setPageSize: React.Dispatch>; - hasPrevPage: boolean; - hasNextPage: boolean; - pushCursor: (nextCursor: string) => void; - popCursor: () => void; - setCursor: (cursor: string | undefined) => void; -} - -const PartitionPagerControls: React.FunctionComponent = ({ - displayed, - pageSize, - setPageSize, - hasNextPage, - hasPrevPage, - setCursor, - pushCursor, - popCursor, -}) => { - return ( - - - {[7, 30, 120].map((size) => ( - - ))} - - - - - - - - - ); -}; - -const PartitionPagerContainer = styled.div` - display: flex; - justify-content: space-between; - margin: 10px 0; -`; - -const PARTITION_SET_QUERY = gql` - query PartitionLongitudinalQuery( - $partitionSetName: String! - $repositorySelector: RepositorySelector! - $partitionsLimit: Int - $partitionsCursor: String - $reverse: Boolean - ) { - partitionSetOrError( - repositorySelector: $repositorySelector - partitionSetName: $partitionSetName - ) { - ... on PartitionSet { - name - partitionsOrError(cursor: $partitionsCursor, limit: $partitionsLimit, reverse: $reverse) { - ... on Partitions { - results { - name - runs { - runId - pipelineName - tags { - key - value - } - stats { - __typename - ... on PipelineRunStatsSnapshot { - startTime - endTime - materializations - } - } - status - stepStats { - __typename - stepKey - startTime - endTime - status - materializations { - __typename - } - expectationResults { - success - } - } - ...RunTableRunFragment - } - } - } - ... on PythonError { - ...PythonErrorFragment - } - } - } - } - } - ${PythonErrorInfo.fragments.PythonErrorFragment} - ${RunTable.fragments.RunTableRunFragment} -`; diff --git a/js_modules/dagit/src/partitions/PartitionsBackfill.tsx b/js_modules/dagit/src/partitions/PartitionsBackfill.tsx --- a/js_modules/dagit/src/partitions/PartitionsBackfill.tsx +++ b/js_modules/dagit/src/partitions/PartitionsBackfill.tsx @@ -46,9 +46,8 @@ export const PartitionsBackfill: React.FunctionComponent<{ partitionSetName: string; pipelineName: string; - showLoader: boolean; onLaunch?: (backfillId: string) => void; -}> = ({partitionSetName, pipelineName, showLoader, onLaunch}) => { +}> = ({partitionSetName, pipelineName, onLaunch}) => { const [isOpen, setOpen] = React.useState(false); return (
{ onLaunch?.(backfillId); setOpen(false); @@ -88,9 +86,8 @@ export const PartitionsBackfillPartitionSelector: React.FunctionComponent<{ partitionSetName: string; pipelineName: string; - showLoader: boolean; onLaunch?: (backfillId: string) => void; -}> = ({partitionSetName, pipelineName, showLoader, onLaunch}) => { +}> = ({partitionSetName, pipelineName, onLaunch}) => { const repositorySelector = useRepositorySelector(); const [currentSelectionRange, setCurrentSelectionRange] = React.useState< SelectionRange | undefined @@ -130,7 +127,7 @@ }, ); - if ((!data || loading) && showLoader) { + if (!data || loading) { return (
> = ({location, match}) => { +}>> = ({match}) => { const {pipelineName, snapshotId} = explorerPathFromString(match.params.pipelinePath); useDocumentTitle(`Pipeline: ${pipelineName}`); @@ -32,15 +30,7 @@ fetchPolicy: 'network-only', skip: !repositorySelector.repositoryLocationName || !repositorySelector.repositoryName, }); - const {history} = React.useContext(RouterContext); - const qs = querystring.parse(location.search); - const cursor = (qs.cursor as string) || undefined; - const setCursor = (cursor: string | undefined) => { - history.push({search: `?${querystring.stringify({...qs, cursor})}`}); - }; - const [selected, setSelected] = React.useState(); - const [showLoader, setShowLoader] = React.useState(false); const [runTags, setRunTags] = React.useState<{[key: string]: string}>({}); if (snapshotId) { @@ -93,15 +83,11 @@ setRunTags({'dagster/backfill': backfillId})} /> setShowLoader(true)} runTags={runTags} /> diff --git a/js_modules/dagit/src/partitions/types/PartitionLongitudinalQuery.ts b/js_modules/dagit/src/partitions/types/PartitionLongitudinalQuery.ts --- a/js_modules/dagit/src/partitions/types/PartitionLongitudinalQuery.ts +++ b/js_modules/dagit/src/partitions/types/PartitionLongitudinalQuery.ts @@ -118,7 +118,7 @@ export interface PartitionLongitudinalQueryVariables { partitionSetName: string; repositorySelector: RepositorySelector; - partitionsLimit?: number | null; - partitionsCursor?: string | null; + limit?: number | null; + cursor?: string | null; reverse?: boolean | null; } diff --git a/js_modules/dagit/src/partitions/useChunkedPartitionsQuery.tsx b/js_modules/dagit/src/partitions/useChunkedPartitionsQuery.tsx new file mode 100644 --- /dev/null +++ b/js_modules/dagit/src/partitions/useChunkedPartitionsQuery.tsx @@ -0,0 +1,197 @@ +import gql from 'graphql-tag'; +import * as React from 'react'; +import {useApolloClient} from 'react-apollo'; + +import {useRepositorySelector} from 'src/DagsterRepositoryContext'; +import {PythonErrorInfo} from 'src/PythonErrorInfo'; +import { + PartitionLongitudinalQuery, + PartitionLongitudinalQueryVariables, + PartitionLongitudinalQuery_partitionSetOrError_PartitionSet_partitionsOrError_Partitions_results, +} from 'src/partitions/types/PartitionLongitudinalQuery'; +import {RunTable} from 'src/runs/RunTable'; + +type Partition = PartitionLongitudinalQuery_partitionSetOrError_PartitionSet_partitionsOrError_Partitions_results; + +interface DataState { + results: Partition[]; + loading: boolean; + cursorStack: string[]; + cursor: string | null; +} + +const InitialDataState: DataState = {results: [], cursor: null, cursorStack: [], loading: false}; + +/** + * This React hook mirrors `useCursorPaginatedQuery` but collects each page of partitions + * in slices that are smaller than pageSize and cause the results to load incrementally. + */ +export function useChunkedPartitionsQuery(partitionSetName: string, pageSize: number) { + const {repositoryName, repositoryLocationName} = useRepositorySelector(); + const client = useApolloClient(); + + const version = React.useRef(0); + const [dataState, setDataState] = React.useState(InitialDataState); + const {cursor, loading, results, cursorStack} = dataState; + + React.useEffect(() => { + const v = version.current + 1; + version.current = v; + + setDataState((dataState) => ({...dataState, results: [], loading: true})); + + let c = cursor; + let accumulated: Partition[] = []; + const fetchOne = async () => { + const result = await client.query< + PartitionLongitudinalQuery, + PartitionLongitudinalQueryVariables + >({ + fetchPolicy: 'network-only', + query: PARTITION_SET_QUERY, + variables: { + partitionSetName, + repositorySelector: {repositoryName, repositoryLocationName}, + reverse: true, + cursor: c, + limit: Math.min(2, pageSize - accumulated.length), + }, + }); + if (version.current !== v) { + return; + } + const fetched = partitionsFromResult(result.data); + accumulated = [...fetched, ...accumulated]; + const more = accumulated.length < pageSize && fetched.length > 0; + + setDataState((dataState) => ({...dataState, results: accumulated, loading: more})); + + if (more) { + c = accumulated[0].name; + fetchOne(); + } + }; + + fetchOne(); + }, [pageSize, cursor, client, partitionSetName, repositoryName, repositoryLocationName]); + + // Note: cursor === null is page zero and cursors specify subsequent pages. + + return { + loading, + partitions: [...buildEmptyPartitions(pageSize - results.length), ...results], + paginationProps: { + hasPrevCursor: cursor !== null, + hasNextCursor: results.length >= pageSize, + popCursor: () => { + if (cursor === null) { + return; + } + setDataState({ + results: [], + cursor: cursorStack.length ? cursorStack[cursorStack.length - 1] : null, + cursorStack: cursorStack.slice(0, cursorStack.length - 1), + loading: false, + }); + }, + advanceCursor: () => { + setDataState({ + loading: false, + cursorStack: cursor ? [...cursorStack, cursor] : cursorStack, + cursor: results[0].name, + results: [], + }); + }, + reset: () => { + setDataState(InitialDataState); + }, + }, + }; +} + +function buildEmptyPartitions(count: number) { + // Note: Partitions don't have any unique keys beside their names, so we use names + // extensively in our display layer as React keys. To create unique empty partitions + // we use different numbers of zero-width space characters + const empty: Partition[] = []; + for (let ii = 0; ii < count; ii++) { + empty.push({ + __typename: 'Partition', + name: `\u200b`.repeat(ii + 1), + runs: [], + }); + } + return empty; +} + +function partitionsFromResult(result?: PartitionLongitudinalQuery) { + if (result?.partitionSetOrError.__typename !== 'PartitionSet') { + return []; + } + if (result.partitionSetOrError.partitionsOrError.__typename !== 'Partitions') { + return []; + } + return result.partitionSetOrError.partitionsOrError.results; +} + +const PARTITION_SET_QUERY = gql` + query PartitionLongitudinalQuery( + $partitionSetName: String! + $repositorySelector: RepositorySelector! + $limit: Int + $cursor: String + $reverse: Boolean + ) { + partitionSetOrError( + repositorySelector: $repositorySelector + partitionSetName: $partitionSetName + ) { + ... on PartitionSet { + name + partitionsOrError(cursor: $cursor, limit: $limit, reverse: $reverse) { + ... on Partitions { + results { + name + runs { + runId + pipelineName + tags { + key + value + } + stats { + __typename + ... on PipelineRunStatsSnapshot { + startTime + endTime + materializations + } + } + status + stepStats { + __typename + stepKey + startTime + endTime + status + materializations { + __typename + } + expectationResults { + success + } + } + ...RunTableRunFragment + } + } + } + ... on PythonError { + ...PythonErrorFragment + } + } + } + } + } + ${PythonErrorInfo.fragments.PythonErrorFragment} + ${RunTable.fragments.RunTableRunFragment} +`; diff --git a/js_modules/dagit/src/runs/RunsRoot.tsx b/js_modules/dagit/src/runs/RunsRoot.tsx --- a/js_modules/dagit/src/runs/RunsRoot.tsx +++ b/js_modules/dagit/src/runs/RunsRoot.tsx @@ -5,7 +5,7 @@ import {RouteComponentProps} from 'react-router'; import styled from 'styled-components/macro'; -import {CursorPaginationControls} from 'src/CursorPaginationControls'; +import {CursorPaginationControls} from 'src/CursorControls'; import {ScrollContainer} from 'src/ListComponents'; import {Loading} from 'src/Loading'; import {useDocumentTitle} from 'src/hooks/useDocumentTitle'; diff --git a/js_modules/dagit/src/runs/useCursorPaginatedQuery.tsx b/js_modules/dagit/src/runs/useCursorPaginatedQuery.tsx --- a/js_modules/dagit/src/runs/useCursorPaginatedQuery.tsx +++ b/js_modules/dagit/src/runs/useCursorPaginatedQuery.tsx @@ -4,13 +4,7 @@ import {useQuery} from 'react-apollo'; import {__RouterContext as RouterContext} from 'react-router'; -export interface CursorPaginationProps { - hasPrevPage: boolean; - hasNextPage: boolean; - onPrevPage: () => void; - onNextPage: () => void; - onReset: () => void; -} +import {CursorPaginationProps} from 'src/CursorControls'; interface CursorPaginationQueryVariables { cursor?: string | null; @@ -61,14 +55,14 @@ const resultArray = options.getResultArray(queryResult.data); const paginationProps: CursorPaginationProps = { - hasPrevPage: !!cursor, - hasNextPage: resultArray.length === options.pageSize + 1, - onPrevPage: () => { + hasPrevCursor: !!cursor, + hasNextCursor: resultArray.length === options.pageSize + 1, + popCursor: () => { const nextStack = [...cursorStack]; setCursor(nextStack.pop()); setCursorStack(nextStack); }, - onNextPage: () => { + advanceCursor: () => { if (cursor) { setCursorStack([...cursorStack, cursor]); } @@ -78,7 +72,7 @@ } setCursor(nextCursor); }, - onReset: () => { + reset: () => { setCursorStack([]); setCursor(undefined); },