8

Use trigger for dynamic select test job

prepare_test:
  image: $CI_REGISTRY/platform/docker-images/vault:1.8
  variables:
    CONTEXT_TEST: |
      include:
      # PRODUCT
        - project: 'gitlabci/integration-test'
          ref: dev_v2
          file: 
            - 'spark/.base_integration_test.yml'
            - 'spark/.base_integration_test_with_kafka.yml'   
      integration_test:
        variables:
          COVERAGE_SOURCE: "./src"  
    INTEGRATION_TEST: |
      $CONTEXT_TEST
        extends: .base_integration_test
    INTEGRATION_TEST_WITH_KAFKA: |
      $CONTEXT_TEST 
        extends: .base_integration_test_with_kafka  
  stage: prepare_test
  script:
    - export CICD_KAFKA_HOST=$(cat test/fixtures.py | grep KAFKA_HOST)
    - >
      if [ "$CICD_KAFKA_HOST" != "" ]; then
        export CICD_KAFKA_HOST="true"
        echo "$INTEGRATION_TEST_WITH_KAFKA" >> test.yml
      else
        export CICD_KAFKA_HOST="false"
        echo "$INTEGRATION_TEST" >> test.yml
      fi
    - env | sort -f
  artifacts:
    paths:
      - test.yml
    expire_in: 6000 seconds


# --------------- Integration test --------------- ###
integration_test:
  stage: test 
  trigger:
    include:
      - artifact: test.yml
        job: prepare_test
    strategy: depend

enter image description here

after complete child integration_test create coverage-report.xml

How return coverage-report.xml to parent pipeline?

2 Answers 2

1

You can use Gitlab API:

my-job:
  image: ...
  stage: ...
  script:
    - >
        export CI_CHILD_PIPELINE_ID=$(curl --header "PRIVATE-TOKEN: $GITLAB_USER_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/bridges" | jq ".[].downstream_pipeline.id")
    - echo $CI_CHILD_PIPELINE_ID
    - >
        export CI_CHILD_JOB_ID=$(curl --header "PRIVATE-TOKEN: $GITLAB_USER_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_CHILD_PIPELINE_ID/jobs" | jq '.[].id')
    - echo $CI_CHILD_JOB_ID 
    - 'curl --output artifacts.zip --header "PRIVATE-TOKEN: $GITLAB_USER_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$CI_CHILD_JOB_ID/artifacts"'
    - unzip artifacts.zip
0

In case someone need it, this would download and create folders for every child pipeline under a folder:


# Expose artifacts from the child pipeline
npm-expose-artifacts:
  stage: test
  image: ...
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: always
    - when: never  # If no conditions above are met, don't run
  script:
    - echo "Running npm expose artifacts job"
    - apt-get update && apt-get install -y curl jq unzip fastjar gawk file
    - curl --version
    - jq --version
    - echo "Publishing artifacts"
    - echo "GitLab User Token [REDACTED]"
    - echo "$CI_API_V4_URL"
    - echo "$GITLAB_USER_TOKEN"

    # Fetch the child pipeline ID with added logging
    - echo "Fetching child pipeline ID..."
    - echo "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/bridges"
    - >
      export BRIDGES_RESPONSE=$(curl --silent --header "PRIVATE-TOKEN: $GL_TOKEN"
      "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/bridges")
    - echo "Bridges Response $BRIDGES_RESPONSE"
    - >
      export CI_CHILD_PIPELINE_ID=$(echo "$BRIDGES_RESPONSE" | jq -r ".[0].downstream_pipeline.id" 
      2>/dev/null || echo "null")
    - >
      if [ "$CI_CHILD_PIPELINE_ID" = "null" ] || [ -z "$CI_CHILD_PIPELINE_ID" ]; then
        echo "Error: Child pipeline ID not found or failed to parse response.";
        exit 1;
      fi
    - echo "Child Pipeline ID  $CI_CHILD_PIPELINE_ID"

    # Fetch the job ID from the child pipeline with added logging
    - echo "Fetching child job ID... $CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_CHILD_PIPELINE_ID/jobs"
    - >
      export JOBS_RESPONSE=$(curl --silent --header "PRIVATE-TOKEN: $GL_TOKEN" 
      "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_CHILD_PIPELINE_ID/jobs")
    - echo "Jobs Response  $JOBS_RESPONSE"
    - >
      export CI_CHILD_JOB_ID=$(echo "$JOBS_RESPONSE" | jq -r '.[0].id' 
      2>/dev/null || echo "null")
    - >
      if [ "$CI_CHILD_JOB_ID" = "null" ] || [ -z "$CI_CHILD_JOB_ID" ]; then
        echo "Error: Child job ID not found or failed to parse response.";
        exit 1;
      fi
    - echo "Child Job ID  $CI_CHILD_JOB_ID"


    - >
      export URLS_ARTIFACTS=$(echo "$JOBS_RESPONSE" | jq -r --arg ci_api_v4_url "$CI_API_V4_URL" '
          .[] |
          .project.id as $project_id |
          .name as $name |
          .id as $job_id |
          "\($ci_api_v4_url)/projects/27363/jobs/\($job_id)/artifacts/ \($name)"
      ')
    
    - echo "$URLS_ARTIFACTS"

    - DOWNLOAD_DIR=".artifacts-download"
    - mkdir -p "$DOWNLOAD_DIR"

    - echo "List of URLs to download:"
    - echo "$URLS_ARTIFACTS"
    - cd "$DOWNLOAD_DIR"
    - >
      echo "$URLS_ARTIFACTS" | while IFS= read -r line; do
        ls -la "."
        # Split the line into URL and filename
        # url=$(echo "$line" | awk '{print $1}')
        # filename=$(echo "$line" | awk '{print $2}')
        echo "start loop"
        echo "line is $line"
        line=$(echo "$line" | xargs)
        echo "line is $line"
        read -r url job_name <<< "$line"

        echo "url is $url"
        echo "job_name is $job_name"

          STATUS_CODE=$(curl -H "JOB-TOKEN: $CI_JOB_TOKEN" -o /dev/null -s -w "%{http_code}" "$url")
          if [ "$STATUS_CODE" -eq 404 ]; then
            echo "404 ignoring $job_name"
          else
            mkdir "$job_name"
            cd "$job_name"
            # Download the artifact using curl
            echo "Downloading $filename from $url..."
            # curl -v -L -H "PRIVATE-TOKEN: $GL_TOKEN" "$url&job_token=$CI_JOB_TOKEN"
            echo "$url"
            #curl -L -H "PRIVATE-TOKEN: $GL_TOKEN" "$url" -o "artifacts.zip"
            # curl -L -H "PRIVATE-TOKEN: $CI_JOB_TOKEN" "$url" -o "artifacts.zip"
            curl -L -H "JOB-TOKEN: $CI_JOB_TOKEN" "$url" -o "artifacts.zip"

            ls -la .
            echo "Extracting ZIP  artifacts.zip"
            file artifacts.zip
            jar tf artifacts.zip
            jar xvf artifacts.zip
            ls -la .

            cd ..
          fi

        echo "-----------------------"
      done
    - cd ..
    - ls -la "$DOWNLOAD_DIR"
    - find "$DOWNLOAD_DIR" | sed -e "s/[^-][^\/]*\//  |/g" -e "s/|\([^ ]\)/|-\1/"

    - echo "Listing files..."
    - ls -la
    - find ./.artifacts-download | sed -e "s/[^-][^\/]*\//  |/g" -e "s/|\([^ ]\)/|-\1/"
  needs:
    - npm-create-jobs
  artifacts:
    when: on_success
    paths:
      - .artifacts-download
    reports:
      junit:
        - .artifacts-download/changethis.test.xml 

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.