add add_catalog_integration call even if we have a pre-existing manifest #43050
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # **what?** | |
| # Runs code quality checks, unit tests, integration tests and | |
| # verifies python build on all code commited to the repository. This workflow | |
| # should not require any secrets since it runs for PRs from forked repos. By | |
| # default, secrets are not passed to workflows running from a forked repos. | |
| # **why?** | |
| # Ensure code for dbt meets a certain quality standard. | |
| # **when?** | |
| # This will run for all PRs, when code is pushed to a release | |
| # branch, and when manually triggered. | |
| name: Tests and Code Checks | |
| on: | |
| push: | |
| branches: | |
| - "main" | |
| - "*.latest" | |
| - "releases/*" | |
| pull_request: | |
| merge_group: | |
| types: [checks_requested] | |
| workflow_dispatch: | |
| permissions: read-all | |
| # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| # top-level adjustments can be made here | |
| env: | |
| # number of parallel processes to spawn for python integration testing | |
| PYTHON_INTEGRATION_TEST_WORKERS: 5 | |
| jobs: | |
| code-quality: | |
| name: code-quality | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6 | |
| with: | |
| python-version: "3.10" | |
| - name: Install python dependencies | |
| run: | | |
| python -m pip install --user --upgrade pip | |
| python -m pip --version | |
| python -m pip install hatch | |
| cd core | |
| hatch run setup | |
| - name: Verify dbt installation | |
| run: | | |
| cd core | |
| hatch run dbt --version | |
| - name: Run pre-commit hooks | |
| run: | | |
| cd core | |
| hatch run code-quality | |
| unit: | |
| name: "unit test / python ${{ matrix.python-version }}" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13"] | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install python dependencies | |
| run: | | |
| python -m pip install --user --upgrade pip | |
| python -m pip --version | |
| python -m pip install hatch | |
| hatch --version | |
| - name: Run unit tests | |
| uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3 | |
| with: | |
| timeout_minutes: 10 | |
| max_attempts: 3 | |
| command: cd core && hatch run ci:unit-tests | |
| - name: Get current date | |
| if: always() | |
| id: date | |
| run: | | |
| CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts | |
| echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT | |
| - name: Upload Unit Test Coverage to Codecov | |
| if: ${{ matrix.python-version == '3.11' }} | |
| uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # codecov/codecov-action@v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| flags: unit | |
| fail_ci_if_error: false | |
| integration-metadata: | |
| name: integration test metadata generation | |
| runs-on: ubuntu-latest | |
| outputs: | |
| split-groups: ${{ steps.generate-split-groups.outputs.split-groups }} | |
| include: ${{ steps.generate-include.outputs.include }} | |
| steps: | |
| - name: generate split-groups | |
| id: generate-split-groups | |
| run: | | |
| MATRIX_JSON="[" | |
| for B in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do | |
| MATRIX_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${B}") | |
| done | |
| MATRIX_JSON="${MATRIX_JSON//\"\"/\", \"}" | |
| MATRIX_JSON+="]" | |
| echo "split-groups=${MATRIX_JSON}" | |
| echo "split-groups=${MATRIX_JSON}" >> $GITHUB_OUTPUT | |
| - name: generate include | |
| id: generate-include | |
| run: | | |
| INCLUDE=('"python-version":"3.10","os":"windows-latest"' '"python-version":"3.10","os":"macos-14"' ) | |
| INCLUDE_GROUPS="[" | |
| for include in ${INCLUDE[@]}; do | |
| for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do | |
| INCLUDE_GROUPS+=$(sed 's/$/, /' <<< "{\"split-group\":\"${group}\",${include}}") | |
| done | |
| done | |
| INCLUDE_GROUPS=$(echo $INCLUDE_GROUPS | sed 's/,*$//g') | |
| INCLUDE_GROUPS+="]" | |
| echo "include=${INCLUDE_GROUPS}" | |
| echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT | |
| integration-postgres: | |
| name: "(${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}" | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| needs: | |
| - integration-metadata | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13"] | |
| os: ["ubuntu-latest"] | |
| split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }} | |
| env: | |
| DBT_INVOCATION_ENV: github-actions | |
| DBT_TEST_USER_1: dbt_test_user_1 | |
| DBT_TEST_USER_2: dbt_test_user_2 | |
| DBT_TEST_USER_3: dbt_test_user_3 | |
| DD_CIVISIBILITY_AGENTLESS_ENABLED: true | |
| DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} | |
| DD_SITE: datadoghq.com | |
| DD_ENV: ci | |
| DD_SERVICE: ${{ github.event.repository.name }} | |
| services: | |
| # Label used to access the service container | |
| postgres: | |
| # Docker Hub image | |
| image: postgres | |
| # Provide the password for postgres | |
| env: | |
| POSTGRES_PASSWORD: password | |
| POSTGRES_USER: postgres | |
| # Set health checks to wait until postgres has started | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Run postgres setup script | |
| run: | | |
| ./test/setup_db.sh | |
| env: | |
| PGHOST: localhost | |
| PGPORT: 5432 | |
| PGPASSWORD: password | |
| - name: Install python tools | |
| run: | | |
| python -m pip install --user --upgrade pip | |
| python -m pip --version | |
| python -m pip install hatch | |
| hatch --version | |
| - name: Run integration tests | |
| uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3 | |
| with: | |
| timeout_minutes: 30 | |
| max_attempts: 3 | |
| shell: bash | |
| command: cd core && hatch run ci:integration-tests -- --ddtrace --splits ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }} --group ${{ matrix.split-group }} | |
| - name: Get current date | |
| if: always() | |
| id: date | |
| run: | | |
| CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts | |
| echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT | |
| - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }} | |
| path: ./logs | |
| - name: Upload Integration Test Coverage to Codecov | |
| if: ${{ matrix.python-version == '3.11' }} | |
| uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # codecov/codecov-action@v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| flags: integration | |
| fail_ci_if_error: false | |
| integration-mac-windows: | |
| name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }} | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| needs: | |
| - integration-metadata | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # already includes split group and runs mac + windows | |
| include: ${{ fromJson(needs.integration-metadata.outputs.include) }} | |
| env: | |
| DBT_INVOCATION_ENV: github-actions | |
| DBT_TEST_USER_1: dbt_test_user_1 | |
| DBT_TEST_USER_2: dbt_test_user_2 | |
| DBT_TEST_USER_3: dbt_test_user_3 | |
| DD_CIVISIBILITY_AGENTLESS_ENABLED: true | |
| DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} | |
| DD_SITE: datadoghq.com | |
| DD_ENV: ci | |
| DD_SERVICE: ${{ github.event.repository.name }} | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up postgres (macos) | |
| if: runner.os == 'macOS' | |
| uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3 | |
| with: | |
| timeout_minutes: 10 | |
| max_attempts: 3 | |
| command: ./test/setup_db.sh | |
| - name: Set up postgres (windows) | |
| if: runner.os == 'Windows' | |
| uses: ./.github/actions/setup-postgres-windows | |
| - name: Install python tools | |
| run: | | |
| python -m pip install --user --upgrade pip | |
| python -m pip --version | |
| python -m pip install hatch | |
| hatch --version | |
| - name: Run integration tests | |
| uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3 | |
| with: | |
| timeout_minutes: 30 | |
| max_attempts: 3 | |
| shell: bash | |
| command: cd core && hatch run ci:integration-tests -- --ddtrace --splits ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }} --group ${{ matrix.split-group }} | |
| - name: Get current date | |
| if: always() | |
| id: date | |
| run: | | |
| CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts | |
| echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT | |
| - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }} | |
| path: ./logs | |
| - name: Upload Integration Test Coverage to Codecov | |
| if: ${{ matrix.python-version == '3.11' }} | |
| uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # codecov/codecov-action@v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| flags: integration | |
| fail_ci_if_error: false | |
| integration-report: | |
| if: ${{ always() }} | |
| name: Integration Test Suite | |
| runs-on: ubuntu-latest | |
| needs: [integration-mac-windows, integration-postgres] | |
| steps: | |
| - name: "Integration Tests Failed" | |
| if: ${{ contains(needs.integration-mac-windows.result, 'failure') || contains(needs.integration-mac-windows.result, 'cancelled') || contains(needs.integration-postgres.result, 'failure') || contains(needs.integration-postgres.result, 'cancelled') }} | |
| # when this is true the next step won't execute | |
| run: | | |
| echo "::notice title='Integration test suite failed'" | |
| exit 1 | |
| - name: "Integration Tests Passed" | |
| run: | | |
| echo "::notice title='Integration test suite passed'" | |
| build: | |
| name: build packages | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6 | |
| with: | |
| python-version: "3.10" | |
| - name: Install python dependencies | |
| run: | | |
| python -m pip install --user --upgrade pip | |
| python -m pip install --upgrade hatch twine check-wheel-contents | |
| python -m pip --version | |
| - name: Build distributions | |
| run: ./scripts/build-dist.sh | |
| - name: Show distributions | |
| run: ls -lh dist/ | |
| - name: Check and verify distributions | |
| run: | | |
| cd core | |
| hatch run build:check-all |