diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -117,6 +117,13 @@ r"aws s3 cp s3://\${BUILDKITE_SECRETS_BUCKET}/gcp-key-elementl-dev.json " + GCP_CREDS_LOCAL_FILE, "export GOOGLE_APPLICATION_CREDENTIALS=" + GCP_CREDS_LOCAL_FILE, + "pushd python_modules/libraries/dagster-airflow/dagster_airflow_tests/", + "docker-compose up -d --remove-orphans", + network_buildkite_container("postgres"), + connect_sibling_docker_container( + "postgres", "test-postgres-db-airflow", "POSTGRES_TEST_DB_HOST", + ), + "popd", ] @@ -152,6 +159,18 @@ ] +def celery_docker_extra_cmds_fn(version): + return celery_extra_cmds_fn(version) + [ + "pushd python_modules/libraries/dagster-celery-docker/dagster_celery_docker_tests/", + "docker-compose up -d --remove-orphans", + network_buildkite_container("postgres"), + connect_sibling_docker_container( + "postgres", "test-postgres-db-celery-docker", "POSTGRES_TEST_DB_HOST", + ), + "popd", + ] + + def integration_suite_extra_cmds_fn(version): return [ 'export AIRFLOW_HOME="/airflow"', @@ -271,84 +290,12 @@ # Some Dagster packages have more involved test configs or support only certain Python version; # special-case those here DAGSTER_PACKAGES_WITH_CUSTOM_TESTS = [ - # Examples: Airline Demo - ModuleBuildSpec( - "examples/airline_demo", - supported_pythons=SupportedPython3s, - extra_cmds_fn=airline_demo_extra_cmds_fn, - buildkite_label="airline-demo", - ), - # Examples: Events Demo - # TODO: https://github.com/dagster-io/dagster/issues/2617 - # ModuleBuildSpec( - # 'examples', - # env_vars=['AWS_SECRET_ACCESS_KEY', 'AWS_ACCESS_KEY_ID', 'AWS_DEFAULT_REGION'], - # supported_pythons=SupportedPython3s, - # tox_file='tox_events.ini', - # buildkite_label='events-demo', - # ), - # Examples - ModuleBuildSpec( - "examples/legacy_examples", - supported_pythons=SupportedPython3s, - extra_cmds_fn=legacy_examples_extra_cmds_fn, - ), - ModuleBuildSpec( - "examples/docs_snippets", - extra_cmds_fn=legacy_examples_extra_cmds_fn, - upload_coverage=False, - supported_pythons=SupportedPython3s, - ), - ModuleBuildSpec("python_modules/dagit", extra_cmds_fn=dagit_extra_cmds_fn), - ModuleBuildSpec("python_modules/automation", supported_pythons=SupportedPython3s), - ModuleBuildSpec( - "python_modules/dagster", - extra_cmds_fn=dagster_extra_cmds_fn, - env_vars=["AWS_ACCOUNT_ID"], - depends_on_fn=test_image_depends_fn, - tox_env_suffixes=[ - "-api_tests", - "-cli_tests", - "-cli_command_cli_api_tests", - "-cli_command_grpc_tests", - "-core_tests", - "-general_tests", - "-scheduler_tests", - ], - ), - ModuleBuildSpec( - "python_modules/dagster-graphql", - tox_env_suffixes=[ - "-in_memory_instance_hosted_user_process_env", - "-in_memory_instance_out_of_process_env", - "-in_memory_instance_multi_location", - "-in_memory_instance_managed_grpc_env", - "-sqlite_instance_hosted_user_process_env", - "-readonly_sqlite_instance_out_of_process_env", - "-sqlite_with_default_run_launcher_out_of_process_env", - "-sqlite_instance_multi_location", - "-sqlite_instance_managed_grpc_env", - "-sqlite_instance_deployed_grpc_env", - ], - ), ModuleBuildSpec( "python_modules/dagster-graphql", extra_cmds_fn=graphql_pg_extra_cmds_fn, tox_file="tox_postgres.ini", buildkite_label="dagster-graphql-postgres", - tox_env_suffixes=[ - "-not_graphql_context_test_suite", - "-postgres_instance_hosted_user_process_env", - "-postgres_instance_out_of_process_env", - "-postgres_instance_multi_location", - "-postgres_instance_managed_grpc_env", - "-postgres_instance_deployed_grpc_env", - ], - ), - ModuleBuildSpec( - "python_modules/libraries/dagster-dbt", - supported_pythons=SupportedPython3s, - extra_cmds_fn=dbt_extra_cmds_fn, + tox_env_suffixes=["-postgres_instance_hosted_user_process_env",], ), ModuleBuildSpec( "python_modules/libraries/dagster-airflow", @@ -362,70 +309,14 @@ ], extra_cmds_fn=airflow_extra_cmds_fn, depends_on_fn=test_image_depends_fn, - tox_env_suffixes=["-default", "-requiresairflowdb"], - ), - ModuleBuildSpec( - "python_modules/libraries/dagster-aws", - env_vars=["AWS_DEFAULT_REGION", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"], - ), - ModuleBuildSpec( - "python_modules/libraries/dagster-azure", env_vars=["AZURE_STORAGE_ACCOUNT_KEY"], - ), - ModuleBuildSpec( - "python_modules/libraries/dagster-celery", - env_vars=["AWS_ACCOUNT_ID", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"], - extra_cmds_fn=celery_extra_cmds_fn, - depends_on_fn=test_image_depends_fn, + tox_env_suffixes=["-requiresairflowdb"], ), ModuleBuildSpec( "python_modules/libraries/dagster-celery-docker", env_vars=["AWS_ACCOUNT_ID", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"], - extra_cmds_fn=celery_extra_cmds_fn, + extra_cmds_fn=celery_docker_extra_cmds_fn, depends_on_fn=test_image_depends_fn, ), - ModuleBuildSpec( - "python_modules/libraries/dagster-dask", - env_vars=["AWS_SECRET_ACCESS_KEY", "AWS_ACCESS_KEY_ID", "AWS_DEFAULT_REGION"], - supported_pythons=SupportedPython3s, - ), - ModuleBuildSpec("python_modules/libraries/dagster-flyte", supported_pythons=SupportedPython3s), - ModuleBuildSpec( - "python_modules/libraries/dagster-gcp", - env_vars=[ - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "BUILDKITE_SECRETS_BUCKET", - "GCP_PROJECT_ID", - ], - extra_cmds_fn=gcp_extra_cmds_fn, - # Remove once https://github.com/dagster-io/dagster/issues/2511 is resolved - retries=2, - ), - ModuleBuildSpec("python_modules/libraries/dagster-ge", supported_pythons=SupportedPython3s), - ModuleBuildSpec( - "python_modules/libraries/dagster-k8s", - env_vars=[ - "AWS_ACCOUNT_ID", - "AWS_ACCESS_KEY_ID", - "AWS_SECRET_ACCESS_KEY", - "BUILDKITE_SECRETS_BUCKET", - ], - extra_cmds_fn=k8s_extra_cmds_fn, - depends_on_fn=test_image_depends_fn, - ), - ModuleBuildSpec( - "python_modules/libraries/dagster-postgres", extra_cmds_fn=postgres_extra_cmds_fn - ), - ModuleBuildSpec( - "python_modules/libraries/dagster-twilio", - env_vars=["TWILIO_TEST_ACCOUNT_SID", "TWILIO_TEST_AUTH_TOKEN"], - # Remove once https://github.com/dagster-io/dagster/issues/2511 is resolved - retries=2, - ), - ModuleBuildSpec( - "python_modules/libraries/dagster-pyspark", supported_pythons=SupportedPython3s - ), - ModuleBuildSpec("python_modules/libraries/lakehouse", supported_pythons=SupportedPython3s), ] @@ -658,61 +549,9 @@ steps = [] steps += publish_test_images() - steps += [ - StepBuilder("dagster-flyte build example") - .run("cd python_modules/libraries/dagster-flyte/examples", "make docker_build") - .on_integration_image(SupportedPython.V3_6) - .build() - ] - - steps += pylint_steps() - steps += [ - StepBuilder("isort") - .run("pip install isort>=4.3.21", "make isort", "git diff --exit-code",) - .on_integration_image(SupportedPython.V3_7) - .build(), - StepBuilder("black") - # See: https://github.com/dagster-io/dagster/issues/1999 - .run("make check_black").on_integration_image(SupportedPython.V3_7).build(), - StepBuilder("docs snapshot test") - .run( - "pip install -r docs-requirements.txt -qqq", - "pip install -r python_modules/dagster/dev-requirements.txt -qqq", - "pip install -e python_modules/dagster -qqq", - "pip install -e python_modules/libraries/dagstermill -qqq", - "pytest -vv docs/test_doc_build.py", - ) - .on_integration_image(SupportedPython.V3_7) - .build(), - StepBuilder("mypy examples") - .run( - "pip install mypy", - # start small by making sure the local code type checks - "mypy examples/airline_demo/airline_demo " - "examples/legacy_examples/dagster_examples/bay_bikes " - "examples/docs_snippets/docs_snippets/intro_tutorial/basics/e04_quality/custom_types_mypy* " - "--ignore-missing-imports", - ) - .on_integration_image(SupportedPython.V3_7) - .build(), - StepBuilder("Validate Library Docs") - .run("pip install -e python_modules/automation", "dagster-docs validate-libraries") - .on_integration_image(SupportedPython.V3_7) - .build(), - ] - for m in DAGSTER_PACKAGES_WITH_CUSTOM_TESTS: steps += m.get_tox_build_steps() - steps += extra_library_tests() - - # https://github.com/dagster-io/dagster/issues/2785 - steps += pipenv_smoke_tests() - steps += version_equality_checks() - steps += next_docs_build_tests() - steps += examples_tests() - steps += integration_tests() - return steps diff --git a/examples/airline_demo/tox.ini b/examples/airline_demo/tox.ini --- a/examples/airline_demo/tox.ini +++ b/examples/airline_demo/tox.ini @@ -21,7 +21,7 @@ -r ../../python_modules/dagster/dev-requirements.txt -r ./dev-requirements.txt -passenv = CI_* COVERALLS_REPO_TOKEN TOXENV SLUGIFY_USES_TEXT_UNIDECODE AWS_* JAVA_HOME POSTGRES_TEST_DB_HOST +passenv = CI_* COVERALLS_REPO_TOKEN TOXENV SLUGIFY_USES_TEXT_UNIDECODE AWS_* JAVA_HOME whitelist_externals = /bin/bash echo diff --git a/python_modules/dagster/dagster/utils/test/postgres_instance.py b/python_modules/dagster/dagster/utils/test/postgres_instance.py --- a/python_modules/dagster/dagster/utils/test/postgres_instance.py +++ b/python_modules/dagster/dagster/utils/test/postgres_instance.py @@ -102,6 +102,12 @@ from dagster_postgres.utils import wait_for_connection # pylint: disable=import-error if BUILDKITE: + + print( + "I AM ASSUMING THIS IS ALREADY UP, IS IT???? " + + repr(TestPostgresInstance.conn_string(**conn_args)) + ) + yield TestPostgresInstance.conn_string( **conn_args ) # buildkite docker is handled in pipeline setup diff --git a/python_modules/libraries/dagster-airflow/tox.ini b/python_modules/libraries/dagster-airflow/tox.ini --- a/python_modules/libraries/dagster-airflow/tox.ini +++ b/python_modules/libraries/dagster-airflow/tox.ini @@ -2,7 +2,7 @@ envlist = py{38,37,36,27}-{unix,windows}-{default,requiresairflowdb},pylint [testenv] -passenv = HOME AIRFLOW_HOME AWS_* BUILDKITE CI_* COVERALLS_REPO_TOKEN DAGSTER_* DOCKER_* GOOGLE_* KUBECONFIG +passenv = HOME AIRFLOW_HOME AWS_* BUILDKITE CI_* COVERALLS_REPO_TOKEN DAGSTER_* DOCKER_* GOOGLE_* KUBECONFIG, POSTGRES_TEST_DB_HOST setenv = SLUGIFY_USES_TEXT_UNIDECODE = yes VIRTUALENV_NO_DOWNLOAD = yes @@ -28,7 +28,7 @@ requiresairflowdb: airflow initdb echo -e "--- \033[0;32m:pytest: Running tox tests\033[0m" !requiresairflowdb: pytest -m "not requires_airflow_db" -s -vv --junitxml=test_results.xml --cov=dagster_airflow --cov-append --cov-report= {posargs} - requiresairflowdb: pytest -m requires_airflow_db -s -vv --junitxml=test_results.xml --cov=dagster_airflow --cov-append --cov-report= {posargs} + requiresairflowdb: pytest -m requires_airflow_db -s -vv --junitxml=test_results.xml --cov=dagster_airflow --cov-append --cov-report= -k test_error_dag_containerized {posargs} coverage report --omit='.tox/*,**/test_*.py' --skip-covered coverage html --omit='.tox/*,**/test_*.py' coverage xml --omit='.tox/*,**/test_*.py' diff --git a/python_modules/libraries/dagster-celery-docker/dagster_celery_docker_tests/test_execute_docker.py b/python_modules/libraries/dagster-celery-docker/dagster_celery_docker_tests/test_execute_docker.py --- a/python_modules/libraries/dagster-celery-docker/dagster_celery_docker_tests/test_execute_docker.py +++ b/python_modules/libraries/dagster-celery-docker/dagster_celery_docker_tests/test_execute_docker.py @@ -97,30 +97,30 @@ except docker.errors.ImageNotFound: build_and_tag_test_image(docker_image) - run_config = merge_dicts( - merge_yamls( - [ - os.path.join(test_project_environments_path(), "env.yaml"), - os.path.join(test_project_environments_path(), "env_s3.yaml"), - ] - ), - { - "execution": { - "celery-docker": { - "config": { - "docker": docker_config, - "config_source": {"task_always_eager": True}, - } + run_config = merge_dicts( + merge_yamls( + [ + os.path.join(test_project_environments_path(), "env.yaml"), + os.path.join(test_project_environments_path(), "env_s3.yaml"), + ] + ), + { + "execution": { + "celery-docker": { + "config": { + "docker": docker_config, + "config_source": {"task_always_eager": True}, } - }, + } }, - ) + }, + ) - with postgres_instance() as instance: + with postgres_instance() as instance: - result = execute_pipeline( - get_test_project_recon_pipeline("docker_celery_pipeline"), - run_config=run_config, - instance=instance, - ) - assert result.success + result = execute_pipeline( + get_test_project_recon_pipeline("docker_celery_pipeline"), + run_config=run_config, + instance=instance, + ) + assert result.success diff --git a/python_modules/libraries/dagster-celery-docker/tox.ini b/python_modules/libraries/dagster-celery-docker/tox.ini --- a/python_modules/libraries/dagster-celery-docker/tox.ini +++ b/python_modules/libraries/dagster-celery-docker/tox.ini @@ -2,7 +2,7 @@ envlist = py{38,37,36,27}-{unix,windows},pylint [testenv] -passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID DAGSTER_DOCKER_* +passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID DAGSTER_DOCKER_* POSTGRES_TEST_DB_HOST deps = -e ../../dagster -e ../../dagster-graphql @@ -22,7 +22,7 @@ !windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster -e dagit' coverage erase echo -e "--- \033[0;32m:pytest: Running tox tests\033[0m" - pytest -vv --junitxml=test_results.xml --cov=dagster_celery_docker --cov-append --cov-report= {posargs} + pytest -vv --junitxml=test_results.xml --cov=dagster_celery_docker --cov-append --cov-report= -s -k test_execute_celery_docker {posargs} coverage report --omit='.tox/*,**/test_*.py' --skip-covered coverage html --omit='.tox/*,**/test_*.py' coverage xml --omit='.tox/*,**/test_*.py'