config.yaml 

inputs:
  - name: aut_logs
    format: avro
    locations:
      - folder: s3://c901.default@com.aqfer.c901.input/aut/avro/runid=20*/

transform:
  - convert_field:
      field_name: user_agent
      new_field_name: user_agent_unwrapped_components
      conversion:
        to_user_agent_struct:
          cache_file: s3://c901.default@com.aqfer.c901.development/vitor_test/aut_split_job/cache/part-00000-00000.avro
  - python:
      file: '{{env "USER_DIR"}}/transform.py'

outputs:
  - destination: s3://c901.default@com.aqfer.c901.development/vitor_test/aut_split_job/output/
    format: avro
    split_by:
      key:
        - event_type
        - run_id
      strip_key: true

  • Table of contents

Was this article helpful?