data_channels.yaml

~~cid~~:
  tmp:
    type: s3aq
    format: parquet
    bucket: com.aqfer..development
    base_path: "/new-pipeline-test/tmp"
    pattern: "*"
    credentials: default
  client_activity_input:
    base_path: /new-pipeline-test/data-validation/input/activity/1/
    bucket: com.aqfer..development
    credentials: default
    format: csv
    options:
      has_header: true
      is_compressed: true      
    pattern: dcm_account428001_activity_202205*gz
    type: s3aq  
  gcm_clicks:
    base_path: /new-pipeline-test/data-validation/input/clicks/1/
    bucket: com.aqfer..development
    credentials: default
    format: csv
    options:
      has_header: true
      is_compressed: true
    pattern: dcm_account428001_click_202205*gz
    type: s3aq
  gcm_impressions:
    base_path: /new-pipeline-test/data-validation/input/impressions/1/
    bucket: com.aqfer..development
    credentials: default
    format: csv
    options:
      has_header: true
      is_compressed: true
    pattern: dcm_account428001_impression_202205*gz
    type: s3aq    
  aqfer_quarentine:
    base_path: /new-pipeline-test/retest-click/output/aqfer_quarentine
    type: s3aq
    format: avro
    bucket: com.aqfer..development
    pattern: "*"
    credentials: default
  aqfer_conversion:
    base_path: /new-pipeline-test/retest-click/output/aqfer_conversion/valid_data=true
    type: s3aq
    format: avro
    bucket: com.aqfer..development
    pattern: "*"
    credentials: default
  aqfer_conversion_invalid:
    base_path: /new-pipeline-test/retest-click2/output/aqfer_conversion/valid_data=false
    type: s3aq
    format: avro
    bucket: com.aqfer..development
    pattern: "*"
    credentials: default
  skipped_file:
    base_path: /new-pipeline-test/retest-click/output/skipped2/
    type: s3aq
    format: parquet
    bucket: com.aqfer..development
    pattern: "*"
    credentials: default    
  aqfer_impression:
    base_path: /new-pipeline-test/retest-click/output/aqfer_impression/valid_data=true
    type: s3aq
    format: avro
    bucket: com.aqfer..development
    pattern: "*"
    credentials: default
  aqfer_impression_invalid:
    base_path: /new-pipeline-test/retest-click/output/aqfer_impression/valid_data=false
    type: s3aq
    format: avro
    bucket: com.aqfer..development
    pattern: "*"
    credentials: default
  aqfer_click_subevent_ext:
    base_path: /new-pipeline-test/retest-click/output/aqfer_click_subevent_ext/valid_data=true
    type: s3aq
    format: avro
    bucket: com.aqfer..development
    pattern: "*"
    credentials: default
  aqfer_click_subevent_ext_invalid:
    base_path: /new-pipeline-test/retest-click/output/aqfer_click_subevent_ext/valid_data=false
    type: s3aq
    format: avro
    bucket: com.aqfer..development
    pattern: "*"
    credentials: default
  • Table of contents

Was this article helpful?