~~cid~~:
tmp:
type: s3aq
format: parquet
bucket: com.aqfer..development
base_path: "/new-pipeline-test/tmp"
pattern: "*"
credentials: default
client_activity_input:
base_path: /new-pipeline-test/data-validation/input/activity/1/
bucket: com.aqfer..development
credentials: default
format: csv
options:
has_header: true
is_compressed: true
pattern: dcm_account428001_activity_202205*gz
type: s3aq
gcm_clicks:
base_path: /new-pipeline-test/data-validation/input/clicks/1/
bucket: com.aqfer..development
credentials: default
format: csv
options:
has_header: true
is_compressed: true
pattern: dcm_account428001_click_202205*gz
type: s3aq
gcm_impressions:
base_path: /new-pipeline-test/data-validation/input/impressions/1/
bucket: com.aqfer..development
credentials: default
format: csv
options:
has_header: true
is_compressed: true
pattern: dcm_account428001_impression_202205*gz
type: s3aq
aqfer_quarentine:
base_path: /new-pipeline-test/retest-click/output/aqfer_quarentine
type: s3aq
format: avro
bucket: com.aqfer..development
pattern: "*"
credentials: default
aqfer_conversion:
base_path: /new-pipeline-test/retest-click/output/aqfer_conversion/valid_data=true
type: s3aq
format: avro
bucket: com.aqfer..development
pattern: "*"
credentials: default
aqfer_conversion_invalid:
base_path: /new-pipeline-test/retest-click2/output/aqfer_conversion/valid_data=false
type: s3aq
format: avro
bucket: com.aqfer..development
pattern: "*"
credentials: default
skipped_file:
base_path: /new-pipeline-test/retest-click/output/skipped2/
type: s3aq
format: parquet
bucket: com.aqfer..development
pattern: "*"
credentials: default
aqfer_impression:
base_path: /new-pipeline-test/retest-click/output/aqfer_impression/valid_data=true
type: s3aq
format: avro
bucket: com.aqfer..development
pattern: "*"
credentials: default
aqfer_impression_invalid:
base_path: /new-pipeline-test/retest-click/output/aqfer_impression/valid_data=false
type: s3aq
format: avro
bucket: com.aqfer..development
pattern: "*"
credentials: default
aqfer_click_subevent_ext:
base_path: /new-pipeline-test/retest-click/output/aqfer_click_subevent_ext/valid_data=true
type: s3aq
format: avro
bucket: com.aqfer..development
pattern: "*"
credentials: default
aqfer_click_subevent_ext_invalid:
base_path: /new-pipeline-test/retest-click/output/aqfer_click_subevent_ext/valid_data=false
type: s3aq
format: avro
bucket: com.aqfer..development
pattern: "*"
credentials: default